block/throttle-groups.c

   1 /*
   2  * QEMU block throttling group infrastructure
   3  *
   4  * Copyright (C) Nodalink, EURL. 2014
   5  * Copyright (C) Igalia, S.L. 2015
   6  *
   7  * Authors:
   8  *   Benoît Canet <benoit.canet@nodalink.com>
   9  *   Alberto Garcia <berto@igalia.com>
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU General Public License as
  13  * published by the Free Software Foundation; either version 2 or
  14  * (at your option) version 3 of the License.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "sysemu/block-backend.h"
  27 #include "block/throttle-groups.h"
  28 #include "qemu/throttle-options.h"
  29 #include "qemu/main-loop.h"
  30 #include "qemu/queue.h"
  31 #include "qemu/thread.h"
  32 #include "sysemu/qtest.h"
  33 #include "qapi/error.h"
  34 #include "qapi/qapi-visit-block-core.h"
  35 #include "qom/object.h"
  36 #include "qom/object_interfaces.h"
  37
  38 static void throttle_group_obj_init(Object *obj);
  39 static void throttle_group_obj_complete(UserCreatable *obj, Error **errp);
  40 static void timer_cb(ThrottleGroupMember *tgm, bool is_write);
  41
  42 /* The ThrottleGroup structure (with its ThrottleState) is shared
  43  * among different ThrottleGroupMembers and it's independent from
  44  * AioContext, so in order to use it from different threads it needs
  45  * its own locking.
  46  *
  47  * This locking is however handled internally in this file, so it's
  48  * transparent to outside users.
  49  *
  50  * The whole ThrottleGroup structure is private and invisible to
  51  * outside users, that only use it through its ThrottleState.
  52  *
  53  * In addition to the ThrottleGroup structure, ThrottleGroupMember has
  54  * fields that need to be accessed by other members of the group and
  55  * therefore also need to be protected by this lock. Once a
  56  * ThrottleGroupMember is registered in a group those fields can be accessed
  57  * by other threads any time.
  58  *
  59  * Again, all this is handled internally and is mostly transparent to
  60  * the outside. The 'throttle_timers' field however has an additional
  61  * constraint because it may be temporarily invalid (see for example
  62  * blk_set_aio_context()). Therefore in this file a thread will
  63  * access some other ThrottleGroupMember's timers only after verifying that
  64  * that ThrottleGroupMember has throttled requests in the queue.
  65  */
  66 struct ThrottleGroup {
  67     Object parent_obj;
  68
  69     /* refuse individual property change if initialization is complete */
  70     bool is_initialized;
  71     char *name; /* This is constant during the lifetime of the group */
  72
  73     QemuMutex lock; /* This lock protects the following four fields */
  74     ThrottleState ts;
  75     QLIST_HEAD(, ThrottleGroupMember) head;
  76     ThrottleGroupMember *tokens[2];
  77     bool any_timer_armed[2];
  78     QEMUClockType clock_type;
  79
  80     /* This field is protected by the global QEMU mutex */
  81     QTAILQ_ENTRY(ThrottleGroup) list;
  82 };
  83
  84 /* This is protected by the global QEMU mutex */
  85 static QTAILQ_HEAD(, ThrottleGroup) throttle_groups =
  86     QTAILQ_HEAD_INITIALIZER(throttle_groups);
  87
  88
  89 /* This function reads throttle_groups and must be called under the global
  90  * mutex.
  91  */
  92 static ThrottleGroup *throttle_group_by_name(const char *name)
  93 {
  94     ThrottleGroup *iter;
  95
  96     /* Look for an existing group with that name */
  97     QTAILQ_FOREACH(iter, &throttle_groups, list) {
  98         if (!g_strcmp0(name, iter->name)) {
  99             return iter;
 100         }
 101     }
 102
 103     return NULL;
 104 }
 105
 106 /* This function reads throttle_groups and must be called under the global
 107  * mutex.
 108  */
 109 bool throttle_group_exists(const char *name)
 110 {
 111     return throttle_group_by_name(name) != NULL;
 112 }
 113
 114 /* Increments the reference count of a ThrottleGroup given its name.
 115  *
 116  * If no ThrottleGroup is found with the given name a new one is
 117  * created.
 118  *
 119  * This function edits throttle_groups and must be called under the global
 120  * mutex.
 121  *
 122  * @name: the name of the ThrottleGroup
 123  * @ret:  the ThrottleState member of the ThrottleGroup
 124  */
 125 ThrottleState *throttle_group_incref(const char *name)
 126 {
 127     ThrottleGroup *tg = NULL;
 128
 129     /* Look for an existing group with that name */
 130     tg = throttle_group_by_name(name);
 131
 132     if (tg) {
 133         object_ref(OBJECT(tg));
 134     } else {
 135         /* Create a new one if not found */
 136         /* new ThrottleGroup obj will have a refcnt = 1 */
 137         tg = THROTTLE_GROUP(object_new(TYPE_THROTTLE_GROUP));
 138         tg->name = g_strdup(name);
 139         throttle_group_obj_complete(USER_CREATABLE(tg), &error_abort);
 140     }
 141
 142     return &tg->ts;
 143 }
 144
 145 /* Decrease the reference count of a ThrottleGroup.
 146  *
 147  * When the reference count reaches zero the ThrottleGroup is
 148  * destroyed.
 149  *
 150  * This function edits throttle_groups and must be called under the global
 151  * mutex.
 152  *
 153  * @ts:  The ThrottleGroup to unref, given by its ThrottleState member
 154  */
 155 void throttle_group_unref(ThrottleState *ts)
 156 {
 157     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 158     object_unref(OBJECT(tg));
 159 }
 160
 161 /* Get the name from a ThrottleGroupMember's group. The name (and the pointer)
 162  * is guaranteed to remain constant during the lifetime of the group.
 163  *
 164  * @tgm:  a ThrottleGroupMember
 165  * @ret:  the name of the group.
 166  */
 167 const char *throttle_group_get_name(ThrottleGroupMember *tgm)
 168 {
 169     ThrottleGroup *tg = container_of(tgm->throttle_state, ThrottleGroup, ts);
 170     return tg->name;
 171 }
 172
 173 /* Return the next ThrottleGroupMember in the round-robin sequence, simulating
 174  * a circular list.
 175  *
 176  * This assumes that tg->lock is held.
 177  *
 178  * @tgm: the current ThrottleGroupMember
 179  * @ret: the next ThrottleGroupMember in the sequence
 180  */
 181 static ThrottleGroupMember *throttle_group_next_tgm(ThrottleGroupMember *tgm)
 182 {
 183     ThrottleState *ts = tgm->throttle_state;
 184     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 185     ThrottleGroupMember *next = QLIST_NEXT(tgm, round_robin);
 186
 187     if (!next) {
 188         next = QLIST_FIRST(&tg->head);
 189     }
 190
 191     return next;
 192 }
 193
 194 /*
 195  * Return whether a ThrottleGroupMember has pending requests.
 196  *
 197  * This assumes that tg->lock is held.
 198  *
 199  * @tgm:        the ThrottleGroupMember
 200  * @is_write:   the type of operation (read/write)
 201  * @ret:        whether the ThrottleGroupMember has pending requests.
 202  */
 203 static inline bool tgm_has_pending_reqs(ThrottleGroupMember *tgm,
 204                                         bool is_write)
 205 {
 206     return tgm->pending_reqs[is_write];
 207 }
 208
 209 /* Return the next ThrottleGroupMember in the round-robin sequence with pending
 210  * I/O requests.
 211  *
 212  * This assumes that tg->lock is held.
 213  *
 214  * @tgm:       the current ThrottleGroupMember
 215  * @is_write:  the type of operation (read/write)
 216  * @ret:       the next ThrottleGroupMember with pending requests, or tgm if
 217  *             there is none.
 218  */
 219 static ThrottleGroupMember *next_throttle_token(ThrottleGroupMember *tgm,
 220                                                 bool is_write)
 221 {
 222     ThrottleState *ts = tgm->throttle_state;
 223     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 224     ThrottleGroupMember *token, *start;
 225
 226     /* If this member has its I/O limits disabled then it means that
 227      * it's being drained. Skip the round-robin search and return tgm
 228      * immediately if it has pending requests. Otherwise we could be
 229      * forcing it to wait for other member's throttled requests. */
 230     if (tgm_has_pending_reqs(tgm, is_write) &&
 231         qatomic_read(&tgm->io_limits_disabled)) {
 232         return tgm;
 233     }
 234
 235     start = token = tg->tokens[is_write];
 236
 237     /* get next bs round in round robin style */
 238     token = throttle_group_next_tgm(token);
 239     while (token != start && !tgm_has_pending_reqs(token, is_write)) {
 240         token = throttle_group_next_tgm(token);
 241     }
 242
 243     /* If no IO are queued for scheduling on the next round robin token
 244      * then decide the token is the current tgm because chances are
 245      * the current tgm got the current request queued.
 246      */
 247     if (token == start && !tgm_has_pending_reqs(token, is_write)) {
 248         token = tgm;
 249     }
 250
 251     /* Either we return the original TGM, or one with pending requests */
 252     assert(token == tgm || tgm_has_pending_reqs(token, is_write));
 253
 254     return token;
 255 }
 256
 257 /* Check if the next I/O request for a ThrottleGroupMember needs to be
 258  * throttled or not. If there's no timer set in this group, set one and update
 259  * the token accordingly.
 260  *
 261  * This assumes that tg->lock is held.
 262  *
 263  * @tgm:        the current ThrottleGroupMember
 264  * @is_write:   the type of operation (read/write)
 265  * @ret:        whether the I/O request needs to be throttled or not
 266  */
 267 static bool throttle_group_schedule_timer(ThrottleGroupMember *tgm,
 268                                           bool is_write)
 269 {
 270     ThrottleState *ts = tgm->throttle_state;
 271     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 272     ThrottleTimers *tt = &tgm->throttle_timers;
 273     bool must_wait;
 274
 275     if (qatomic_read(&tgm->io_limits_disabled)) {
 276         return false;
 277     }
 278
 279     /* Check if any of the timers in this group is already armed */
 280     if (tg->any_timer_armed[is_write]) {
 281         return true;
 282     }
 283
 284     must_wait = throttle_schedule_timer(ts, tt, is_write);
 285
 286     /* If a timer just got armed, set tgm as the current token */
 287     if (must_wait) {
 288         tg->tokens[is_write] = tgm;
 289         tg->any_timer_armed[is_write] = true;
 290     }
 291
 292     return must_wait;
 293 }
 294
 295 /* Start the next pending I/O request for a ThrottleGroupMember. Return whether
 296  * any request was actually pending.
 297  *
 298  * @tgm:       the current ThrottleGroupMember
 299  * @is_write:  the type of operation (read/write)
 300  */
 301 static bool coroutine_fn throttle_group_co_restart_queue(ThrottleGroupMember *tgm,
 302                                                          bool is_write)
 303 {
 304     bool ret;
 305
 306     qemu_co_mutex_lock(&tgm->throttled_reqs_lock);
 307     ret = qemu_co_queue_next(&tgm->throttled_reqs[is_write]);
 308     qemu_co_mutex_unlock(&tgm->throttled_reqs_lock);
 309
 310     return ret;
 311 }
 312
 313 /* Look for the next pending I/O request and schedule it.
 314  *
 315  * This assumes that tg->lock is held.
 316  *
 317  * @tgm:       the current ThrottleGroupMember
 318  * @is_write:  the type of operation (read/write)
 319  */
 320 static void schedule_next_request(ThrottleGroupMember *tgm, bool is_write)
 321 {
 322     ThrottleState *ts = tgm->throttle_state;
 323     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 324     bool must_wait;
 325     ThrottleGroupMember *token;
 326
 327     /* Check if there's any pending request to schedule next */
 328     token = next_throttle_token(tgm, is_write);
 329     if (!tgm_has_pending_reqs(token, is_write)) {
 330         return;
 331     }
 332
 333     /* Set a timer for the request if it needs to be throttled */
 334     must_wait = throttle_group_schedule_timer(token, is_write);
 335
 336     /* If it doesn't have to wait, queue it for immediate execution */
 337     if (!must_wait) {
 338         /* Give preference to requests from the current tgm */
 339         if (qemu_in_coroutine() &&
 340             throttle_group_co_restart_queue(tgm, is_write)) {
 341             token = tgm;
 342         } else {
 343             ThrottleTimers *tt = &token->throttle_timers;
 344             int64_t now = qemu_clock_get_ns(tg->clock_type);
 345             timer_mod(tt->timers[is_write], now);
 346             tg->any_timer_armed[is_write] = true;
 347         }
 348         tg->tokens[is_write] = token;
 349     }
 350 }
 351
 352 /* Check if an I/O request needs to be throttled, wait and set a timer
 353  * if necessary, and schedule the next request using a round robin
 354  * algorithm.
 355  *
 356  * @tgm:       the current ThrottleGroupMember
 357  * @bytes:     the number of bytes for this I/O
 358  * @is_write:  the type of operation (read/write)
 359  */
 360 void coroutine_fn throttle_group_co_io_limits_intercept(ThrottleGroupMember *tgm,
 361                                                         int64_t bytes,
 362                                                         bool is_write)
 363 {
 364     bool must_wait;
 365     ThrottleGroupMember *token;
 366     ThrottleGroup *tg = container_of(tgm->throttle_state, ThrottleGroup, ts);
 367
 368     assert(bytes >= 0);
 369
 370     qemu_mutex_lock(&tg->lock);
 371
 372     /* First we check if this I/O has to be throttled. */
 373     token = next_throttle_token(tgm, is_write);
 374     must_wait = throttle_group_schedule_timer(token, is_write);
 375
 376     /* Wait if there's a timer set or queued requests of this type */
 377     if (must_wait || tgm->pending_reqs[is_write]) {
 378         tgm->pending_reqs[is_write]++;
 379         qemu_mutex_unlock(&tg->lock);
 380         qemu_co_mutex_lock(&tgm->throttled_reqs_lock);
 381         qemu_co_queue_wait(&tgm->throttled_reqs[is_write],
 382                            &tgm->throttled_reqs_lock);
 383         qemu_co_mutex_unlock(&tgm->throttled_reqs_lock);
 384         qemu_mutex_lock(&tg->lock);
 385         tgm->pending_reqs[is_write]--;
 386     }
 387
 388     /* The I/O will be executed, so do the accounting */
 389     throttle_account(tgm->throttle_state, is_write, bytes);
 390
 391     /* Schedule the next request */
 392     schedule_next_request(tgm, is_write);
 393
 394     qemu_mutex_unlock(&tg->lock);
 395 }
 396
 397 typedef struct {
 398     ThrottleGroupMember *tgm;
 399     bool is_write;
 400 } RestartData;
 401
 402 static void coroutine_fn throttle_group_restart_queue_entry(void *opaque)
 403 {
 404     RestartData *data = opaque;
 405     ThrottleGroupMember *tgm = data->tgm;
 406     ThrottleState *ts = tgm->throttle_state;
 407     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 408     bool is_write = data->is_write;
 409     bool empty_queue;
 410
 411     empty_queue = !throttle_group_co_restart_queue(tgm, is_write);
 412
 413     /* If the request queue was empty then we have to take care of
 414      * scheduling the next one */
 415     if (empty_queue) {
 416         qemu_mutex_lock(&tg->lock);
 417         schedule_next_request(tgm, is_write);
 418         qemu_mutex_unlock(&tg->lock);
 419     }
 420
 421     g_free(data);
 422
 423     qatomic_dec(&tgm->restart_pending);
 424     aio_wait_kick();
 425 }
 426
 427 static void throttle_group_restart_queue(ThrottleGroupMember *tgm, bool is_write)
 428 {
 429     Coroutine *co;
 430     RestartData *rd = g_new0(RestartData, 1);
 431
 432     rd->tgm = tgm;
 433     rd->is_write = is_write;
 434
 435     /* This function is called when a timer is fired or when
 436      * throttle_group_restart_tgm() is called. Either way, there can
 437      * be no timer pending on this tgm at this point */
 438     assert(!timer_pending(tgm->throttle_timers.timers[is_write]));
 439
 440     qatomic_inc(&tgm->restart_pending);
 441
 442     co = qemu_coroutine_create(throttle_group_restart_queue_entry, rd);
 443     aio_co_enter(tgm->aio_context, co);
 444 }
 445
 446 void throttle_group_restart_tgm(ThrottleGroupMember *tgm)
 447 {
 448     int i;
 449
 450     if (tgm->throttle_state) {
 451         for (i = 0; i < 2; i++) {
 452             QEMUTimer *t = tgm->throttle_timers.timers[i];
 453             if (timer_pending(t)) {
 454                 /* If there's a pending timer on this tgm, fire it now */
 455                 timer_del(t);
 456                 timer_cb(tgm, i);
 457             } else {
 458                 /* Else run the next request from the queue manually */
 459                 throttle_group_restart_queue(tgm, i);
 460             }
 461         }
 462     }
 463 }
 464
 465 /* Update the throttle configuration for a particular group. Similar
 466  * to throttle_config(), but guarantees atomicity within the
 467  * throttling group.
 468  *
 469  * @tgm:    a ThrottleGroupMember that is a member of the group
 470  * @cfg: the configuration to set
 471  */
 472 void throttle_group_config(ThrottleGroupMember *tgm, ThrottleConfig *cfg)
 473 {
 474     ThrottleState *ts = tgm->throttle_state;
 475     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 476     qemu_mutex_lock(&tg->lock);
 477     throttle_config(ts, tg->clock_type, cfg);
 478     qemu_mutex_unlock(&tg->lock);
 479
 480     throttle_group_restart_tgm(tgm);
 481 }
 482
 483 /* Get the throttle configuration from a particular group. Similar to
 484  * throttle_get_config(), but guarantees atomicity within the
 485  * throttling group.
 486  *
 487  * @tgm:    a ThrottleGroupMember that is a member of the group
 488  * @cfg: the configuration will be written here
 489  */
 490 void throttle_group_get_config(ThrottleGroupMember *tgm, ThrottleConfig *cfg)
 491 {
 492     ThrottleState *ts = tgm->throttle_state;
 493     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 494     qemu_mutex_lock(&tg->lock);
 495     throttle_get_config(ts, cfg);
 496     qemu_mutex_unlock(&tg->lock);
 497 }
 498
 499 /* ThrottleTimers callback. This wakes up a request that was waiting
 500  * because it had been throttled.
 501  *
 502  * @tgm:       the ThrottleGroupMember whose request had been throttled
 503  * @is_write:  the type of operation (read/write)
 504  */
 505 static void timer_cb(ThrottleGroupMember *tgm, bool is_write)
 506 {
 507     ThrottleState *ts = tgm->throttle_state;
 508     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 509
 510     /* The timer has just been fired, so we can update the flag */
 511     qemu_mutex_lock(&tg->lock);
 512     tg->any_timer_armed[is_write] = false;
 513     qemu_mutex_unlock(&tg->lock);
 514
 515     /* Run the request that was waiting for this timer */
 516     throttle_group_restart_queue(tgm, is_write);
 517 }
 518
 519 static void read_timer_cb(void *opaque)
 520 {
 521     timer_cb(opaque, false);
 522 }
 523
 524 static void write_timer_cb(void *opaque)
 525 {
 526     timer_cb(opaque, true);
 527 }
 528
 529 /* Register a ThrottleGroupMember from the throttling group, also initializing
 530  * its timers and updating its throttle_state pointer to point to it. If a
 531  * throttling group with that name does not exist yet, it will be created.
 532  *
 533  * This function edits throttle_groups and must be called under the global
 534  * mutex.
 535  *
 536  * @tgm:       the ThrottleGroupMember to insert
 537  * @groupname: the name of the group
 538  * @ctx:       the AioContext to use
 539  */
 540 void throttle_group_register_tgm(ThrottleGroupMember *tgm,
 541                                  const char *groupname,
 542                                  AioContext *ctx)
 543 {
 544     int i;
 545     ThrottleState *ts = throttle_group_incref(groupname);
 546     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 547
 548     tgm->throttle_state = ts;
 549     tgm->aio_context = ctx;
 550     qatomic_set(&tgm->restart_pending, 0);
 551
 552     QEMU_LOCK_GUARD(&tg->lock);
 553     /* If the ThrottleGroup is new set this ThrottleGroupMember as the token */
 554     for (i = 0; i < 2; i++) {
 555         if (!tg->tokens[i]) {
 556             tg->tokens[i] = tgm;
 557         }
 558     }
 559
 560     QLIST_INSERT_HEAD(&tg->head, tgm, round_robin);
 561
 562     throttle_timers_init(&tgm->throttle_timers,
 563                          tgm->aio_context,
 564                          tg->clock_type,
 565                          read_timer_cb,
 566                          write_timer_cb,
 567                          tgm);
 568     qemu_co_mutex_init(&tgm->throttled_reqs_lock);
 569     qemu_co_queue_init(&tgm->throttled_reqs[0]);
 570     qemu_co_queue_init(&tgm->throttled_reqs[1]);
 571 }
 572
 573 /* Unregister a ThrottleGroupMember from its group, removing it from the list,
 574  * destroying the timers and setting the throttle_state pointer to NULL.
 575  *
 576  * The ThrottleGroupMember must not have pending throttled requests, so the
 577  * caller has to drain them first.
 578  *
 579  * The group will be destroyed if it's empty after this operation.
 580  *
 581  * @tgm the ThrottleGroupMember to remove
 582  */
 583 void throttle_group_unregister_tgm(ThrottleGroupMember *tgm)
 584 {
 585     ThrottleState *ts = tgm->throttle_state;
 586     ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
 587     ThrottleGroupMember *token;
 588     int i;
 589
 590     if (!ts) {
 591         /* Discard already unregistered tgm */
 592         return;
 593     }
 594
 595     /* Wait for throttle_group_restart_queue_entry() coroutines to finish */
 596     AIO_WAIT_WHILE(tgm->aio_context, qatomic_read(&tgm->restart_pending) > 0);
 597
 598     WITH_QEMU_LOCK_GUARD(&tg->lock) {
 599         for (i = 0; i < 2; i++) {
 600             assert(tgm->pending_reqs[i] == 0);
 601             assert(qemu_co_queue_empty(&tgm->throttled_reqs[i]));
 602             assert(!timer_pending(tgm->throttle_timers.timers[i]));
 603             if (tg->tokens[i] == tgm) {
 604                 token = throttle_group_next_tgm(tgm);
 605                 /* Take care of the case where this is the last tgm in the group */
 606                 if (token == tgm) {
 607                     token = NULL;
 608                 }
 609                 tg->tokens[i] = token;
 610             }
 611         }
 612
 613         /* remove the current tgm from the list */
 614         QLIST_REMOVE(tgm, round_robin);
 615         throttle_timers_destroy(&tgm->throttle_timers);
 616     }
 617
 618     throttle_group_unref(&tg->ts);
 619     tgm->throttle_state = NULL;
 620 }
 621
 622 void throttle_group_attach_aio_context(ThrottleGroupMember *tgm,
 623                                        AioContext *new_context)
 624 {
 625     ThrottleTimers *tt = &tgm->throttle_timers;
 626     throttle_timers_attach_aio_context(tt, new_context);
 627     tgm->aio_context = new_context;
 628 }
 629
 630 void throttle_group_detach_aio_context(ThrottleGroupMember *tgm)
 631 {
 632     ThrottleGroup *tg = container_of(tgm->throttle_state, ThrottleGroup, ts);
 633     ThrottleTimers *tt = &tgm->throttle_timers;
 634     int i;
 635
 636     /* Requests must have been drained */
 637     assert(tgm->pending_reqs[0] == 0 && tgm->pending_reqs[1] == 0);
 638     assert(qemu_co_queue_empty(&tgm->throttled_reqs[0]));
 639     assert(qemu_co_queue_empty(&tgm->throttled_reqs[1]));
 640
 641     /* Kick off next ThrottleGroupMember, if necessary */
 642     WITH_QEMU_LOCK_GUARD(&tg->lock) {
 643         for (i = 0; i < 2; i++) {
 644             if (timer_pending(tt->timers[i])) {
 645                 tg->any_timer_armed[i] = false;
 646                 schedule_next_request(tgm, i);
 647             }
 648         }
 649     }
 650
 651     throttle_timers_detach_aio_context(tt);
 652     tgm->aio_context = NULL;
 653 }
 654
 655 #undef THROTTLE_OPT_PREFIX
 656 #define THROTTLE_OPT_PREFIX "x-"
 657
 658 /* Helper struct and array for QOM property setter/getter */
 659 typedef struct {
 660     const char *name;
 661     BucketType type;
 662     enum {
 663         AVG,
 664         MAX,
 665         BURST_LENGTH,
 666         IOPS_SIZE,
 667     } category;
 668 } ThrottleParamInfo;
 669
 670 static ThrottleParamInfo properties[] = {
 671     {
 672         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_TOTAL,
 673         THROTTLE_OPS_TOTAL, AVG,
 674     },
 675     {
 676         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_TOTAL_MAX,
 677         THROTTLE_OPS_TOTAL, MAX,
 678     },
 679     {
 680         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_TOTAL_MAX_LENGTH,
 681         THROTTLE_OPS_TOTAL, BURST_LENGTH,
 682     },
 683     {
 684         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_READ,
 685         THROTTLE_OPS_READ, AVG,
 686     },
 687     {
 688         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_READ_MAX,
 689         THROTTLE_OPS_READ, MAX,
 690     },
 691     {
 692         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_READ_MAX_LENGTH,
 693         THROTTLE_OPS_READ, BURST_LENGTH,
 694     },
 695     {
 696         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_WRITE,
 697         THROTTLE_OPS_WRITE, AVG,
 698     },
 699     {
 700         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_WRITE_MAX,
 701         THROTTLE_OPS_WRITE, MAX,
 702     },
 703     {
 704         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_WRITE_MAX_LENGTH,
 705         THROTTLE_OPS_WRITE, BURST_LENGTH,
 706     },
 707     {
 708         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_TOTAL,
 709         THROTTLE_BPS_TOTAL, AVG,
 710     },
 711     {
 712         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_TOTAL_MAX,
 713         THROTTLE_BPS_TOTAL, MAX,
 714     },
 715     {
 716         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_TOTAL_MAX_LENGTH,
 717         THROTTLE_BPS_TOTAL, BURST_LENGTH,
 718     },
 719     {
 720         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_READ,
 721         THROTTLE_BPS_READ, AVG,
 722     },
 723     {
 724         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_READ_MAX,
 725         THROTTLE_BPS_READ, MAX,
 726     },
 727     {
 728         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_READ_MAX_LENGTH,
 729         THROTTLE_BPS_READ, BURST_LENGTH,
 730     },
 731     {
 732         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_WRITE,
 733         THROTTLE_BPS_WRITE, AVG,
 734     },
 735     {
 736         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_WRITE_MAX,
 737         THROTTLE_BPS_WRITE, MAX,
 738     },
 739     {
 740         THROTTLE_OPT_PREFIX QEMU_OPT_BPS_WRITE_MAX_LENGTH,
 741         THROTTLE_BPS_WRITE, BURST_LENGTH,
 742     },
 743     {
 744         THROTTLE_OPT_PREFIX QEMU_OPT_IOPS_SIZE,
 745         0, IOPS_SIZE,
 746     }
 747 };
 748
 749 /* This function edits throttle_groups and must be called under the global
 750  * mutex */
 751 static void throttle_group_obj_init(Object *obj)
 752 {
 753     ThrottleGroup *tg = THROTTLE_GROUP(obj);
 754
 755     tg->clock_type = QEMU_CLOCK_REALTIME;
 756     if (qtest_enabled()) {
 757         /* For testing block IO throttling only */
 758         tg->clock_type = QEMU_CLOCK_VIRTUAL;
 759     }
 760     tg->is_initialized = false;
 761     qemu_mutex_init(&tg->lock);
 762     throttle_init(&tg->ts);
 763     QLIST_INIT(&tg->head);
 764 }
 765
 766 /* This function edits throttle_groups and must be called under the global
 767  * mutex */
 768 static void throttle_group_obj_complete(UserCreatable *obj, Error **errp)
 769 {
 770     ThrottleGroup *tg = THROTTLE_GROUP(obj);
 771     ThrottleConfig cfg;
 772
 773     /* set group name to object id if it exists */
 774     if (!tg->name && tg->parent_obj.parent) {
 775         tg->name = g_strdup(object_get_canonical_path_component(OBJECT(obj)));
 776     }
 777     /* We must have a group name at this point */
 778     assert(tg->name);
 779
 780     /* error if name is duplicate */
 781     if (throttle_group_exists(tg->name)) {
 782         error_setg(errp, "A group with this name already exists");
 783         return;
 784     }
 785
 786     /* check validity */
 787     throttle_get_config(&tg->ts, &cfg);
 788     if (!throttle_is_valid(&cfg, errp)) {
 789         return;
 790     }
 791     throttle_config(&tg->ts, tg->clock_type, &cfg);
 792     QTAILQ_INSERT_TAIL(&throttle_groups, tg, list);
 793     tg->is_initialized = true;
 794 }
 795
 796 /* This function edits throttle_groups and must be called under the global
 797  * mutex */
 798 static void throttle_group_obj_finalize(Object *obj)
 799 {
 800     ThrottleGroup *tg = THROTTLE_GROUP(obj);
 801     if (tg->is_initialized) {
 802         QTAILQ_REMOVE(&throttle_groups, tg, list);
 803     }
 804     qemu_mutex_destroy(&tg->lock);
 805     g_free(tg->name);
 806 }
 807
 808 static void throttle_group_set(Object *obj, Visitor *v, const char * name,
 809                                void *opaque, Error **errp)
 810
 811 {
 812     ThrottleGroup *tg = THROTTLE_GROUP(obj);
 813     ThrottleConfig *cfg;
 814     ThrottleParamInfo *info = opaque;
 815     int64_t value;
 816
 817     /* If we have finished initialization, don't accept individual property
 818      * changes through QOM. Throttle configuration limits must be set in one
 819      * transaction, as certain combinations are invalid.
 820      */
 821     if (tg->is_initialized) {
 822         error_setg(errp, "Property cannot be set after initialization");
 823         return;
 824     }
 825
 826     if (!visit_type_int64(v, name, &value, errp)) {
 827         return;
 828     }
 829     if (value < 0) {
 830         error_setg(errp, "Property values cannot be negative");
 831         return;
 832     }
 833
 834     cfg = &tg->ts.cfg;
 835     switch (info->category) {
 836     case AVG:
 837         cfg->buckets[info->type].avg = value;
 838         break;
 839     case MAX:
 840         cfg->buckets[info->type].max = value;
 841         break;
 842     case BURST_LENGTH:
 843         if (value > UINT_MAX) {
 844             error_setg(errp, "%s value must be in the" "range [0, %u]",
 845                        info->name, UINT_MAX);
 846             return;
 847         }
 848         cfg->buckets[info->type].burst_length = value;
 849         break;
 850     case IOPS_SIZE:
 851         cfg->op_size = value;
 852         break;
 853     }
 854 }
 855
 856 static void throttle_group_get(Object *obj, Visitor *v, const char *name,
 857                                void *opaque, Error **errp)
 858 {
 859     ThrottleGroup *tg = THROTTLE_GROUP(obj);
 860     ThrottleConfig cfg;
 861     ThrottleParamInfo *info = opaque;
 862     int64_t value;
 863
 864     throttle_get_config(&tg->ts, &cfg);
 865     switch (info->category) {
 866     case AVG:
 867         value = cfg.buckets[info->type].avg;
 868         break;
 869     case MAX:
 870         value = cfg.buckets[info->type].max;
 871         break;
 872     case BURST_LENGTH:
 873         value = cfg.buckets[info->type].burst_length;
 874         break;
 875     case IOPS_SIZE:
 876         value = cfg.op_size;
 877         break;
 878     }
 879
 880     visit_type_int64(v, name, &value, errp);
 881 }
 882
 883 static void throttle_group_set_limits(Object *obj, Visitor *v,
 884                                       const char *name, void *opaque,
 885                                       Error **errp)
 886
 887 {
 888     ThrottleGroup *tg = THROTTLE_GROUP(obj);
 889     ThrottleConfig cfg;
 890     ThrottleLimits *argp;
 891     Error *local_err = NULL;
 892
 893     if (!visit_type_ThrottleLimits(v, name, &argp, errp)) {
 894         return;
 895     }
 896     qemu_mutex_lock(&tg->lock);
 897     throttle_get_config(&tg->ts, &cfg);
 898     throttle_limits_to_config(argp, &cfg, &local_err);
 899     if (local_err) {
 900         goto unlock;
 901     }
 902     throttle_config(&tg->ts, tg->clock_type, &cfg);
 903
 904 unlock:
 905     qemu_mutex_unlock(&tg->lock);
 906     qapi_free_ThrottleLimits(argp);
 907     error_propagate(errp, local_err);
 908     return;
 909 }
 910
 911 static void throttle_group_get_limits(Object *obj, Visitor *v,
 912                                       const char *name, void *opaque,
 913                                       Error **errp)
 914 {
 915     ThrottleGroup *tg = THROTTLE_GROUP(obj);
 916     ThrottleConfig cfg;
 917     ThrottleLimits arg = { 0 };
 918     ThrottleLimits *argp = &arg;
 919
 920     qemu_mutex_lock(&tg->lock);
 921     throttle_get_config(&tg->ts, &cfg);
 922     qemu_mutex_unlock(&tg->lock);
 923
 924     throttle_config_to_limits(&cfg, argp);
 925
 926     visit_type_ThrottleLimits(v, name, &argp, errp);
 927 }
 928
 929 static bool throttle_group_can_be_deleted(UserCreatable *uc)
 930 {
 931     return OBJECT(uc)->ref == 1;
 932 }
 933
 934 static void throttle_group_obj_class_init(ObjectClass *klass, void *class_data)
 935 {
 936     size_t i = 0;
 937     UserCreatableClass *ucc = USER_CREATABLE_CLASS(klass);
 938
 939     ucc->complete = throttle_group_obj_complete;
 940     ucc->can_be_deleted = throttle_group_can_be_deleted;
 941
 942     /* individual properties */
 943     for (i = 0; i < sizeof(properties) / sizeof(ThrottleParamInfo); i++) {
 944         object_class_property_add(klass,
 945                                   properties[i].name,
 946                                   "int",
 947                                   throttle_group_get,
 948                                   throttle_group_set,
 949                                   NULL, &properties[i]);
 950     }
 951
 952     /* ThrottleLimits */
 953     object_class_property_add(klass,
 954                               "limits", "ThrottleLimits",
 955                               throttle_group_get_limits,
 956                               throttle_group_set_limits,
 957                               NULL, NULL);
 958 }
 959
 960 static const TypeInfo throttle_group_info = {
 961     .name = TYPE_THROTTLE_GROUP,
 962     .parent = TYPE_OBJECT,
 963     .class_init = throttle_group_obj_class_init,
 964     .instance_size = sizeof(ThrottleGroup),
 965     .instance_init = throttle_group_obj_init,
 966     .instance_finalize = throttle_group_obj_finalize,
 967     .interfaces = (InterfaceInfo[]) {
 968         { TYPE_USER_CREATABLE },
 969         { }
 970     },
 971 };
 972
 973 static void throttle_groups_init(void)
 974 {
 975     type_register_static(&throttle_group_info);
 976 }
 977
 978 type_init(throttle_groups_init);