BFQ_BFQQ_FNS(wait_request);
BFQ_BFQQ_FNS(non_blocking_wait_rq);
BFQ_BFQQ_FNS(fifo_expire);
-BFQ_BFQQ_FNS(idle_window);
+BFQ_BFQQ_FNS(has_short_ttime);
BFQ_BFQQ_FNS(sync);
BFQ_BFQQ_FNS(IO_bound);
BFQ_BFQQ_FNS(in_large_burst);
entity->budget = new_budget;
bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu",
new_budget);
- bfq_requeue_bfqq(bfqd, bfqq);
+ bfq_requeue_bfqq(bfqd, bfqq, false);
}
}
unsigned int old_wr_coeff = bfqq->wr_coeff;
bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq);
- if (bic->saved_idle_window)
- bfq_mark_bfqq_idle_window(bfqq);
+ if (bic->saved_has_short_ttime)
+ bfq_mark_bfqq_has_short_ttime(bfqq);
else
- bfq_clear_bfqq_idle_window(bfqq);
+ bfq_clear_bfqq_has_short_ttime(bfqq);
if (bic->saved_IO_bound)
bfq_mark_bfqq_IO_bound(bfqq);
return;
bic->saved_ttime = bfqq->ttime;
- bic->saved_idle_window = bfq_bfqq_idle_window(bfqq);
+ bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq);
bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);
bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
bfq_del_bfqq_busy(bfqd, bfqq, true);
} else {
- bfq_requeue_bfqq(bfqd, bfqq);
+ bfq_requeue_bfqq(bfqd, bfqq, true);
/*
* Resort priority tree of potential close cooperators.
*/
}
bfq_log_bfqq(bfqd, bfqq,
- "expire (%d, slow %d, num_disp %d, idle_win %d)", reason,
- slow, bfqq->dispatched, bfq_bfqq_idle_window(bfqq));
+ "expire (%d, slow %d, num_disp %d, short_ttime %d)", reason,
+ slow, bfqq->dispatched, bfq_bfqq_has_short_ttime(bfqq));
/*
* Increase, decrease or leave budget unchanged according to
static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
{
struct bfq_data *bfqd = bfqq->bfqd;
- bool idling_boosts_thr, idling_boosts_thr_without_issues,
+ bool rot_without_queueing =
+ !blk_queue_nonrot(bfqd->queue) && !bfqd->hw_tag,
+ bfqq_sequential_and_IO_bound,
+ idling_boosts_thr, idling_boosts_thr_without_issues,
idling_needed_for_service_guarantees,
asymmetric_scenario;
if (bfqd->strict_guarantees)
return true;
+ /*
+ * Idling is performed only if slice_idle > 0. In addition, we
+ * do not idle if
+ * (a) bfqq is async
+ * (b) bfqq is in the idle io prio class: in this case we do
+ * not idle because we want to minimize the bandwidth that
+ * queues in this class can steal to higher-priority queues
+ */
+ if (bfqd->bfq_slice_idle == 0 || !bfq_bfqq_sync(bfqq) ||
+ bfq_class_idle(bfqq))
+ return false;
+
+ bfqq_sequential_and_IO_bound = !BFQQ_SEEKY(bfqq) &&
+ bfq_bfqq_IO_bound(bfqq) && bfq_bfqq_has_short_ttime(bfqq);
+
/*
* The next variable takes into account the cases where idling
* boosts the throughput.
*
* The value of the variable is computed considering, first, that
* idling is virtually always beneficial for the throughput if:
- * (a) the device is not NCQ-capable, or
- * (b) regardless of the presence of NCQ, the device is rotational
- * and the request pattern for bfqq is I/O-bound and sequential.
+ * (a) the device is not NCQ-capable and rotational, or
+ * (b) regardless of the presence of NCQ, the device is rotational and
+ * the request pattern for bfqq is I/O-bound and sequential, or
+ * (c) regardless of whether it is rotational, the device is
+ * not NCQ-capable and the request pattern for bfqq is
+ * I/O-bound and sequential.
*
* Secondly, and in contrast to the above item (b), idling an
* NCQ-capable flash-based device would not boost the
* throughput even with sequential I/O; rather it would lower
* the throughput in proportion to how fast the device
* is. Accordingly, the next variable is true if any of the
- * above conditions (a) and (b) is true, and, in particular,
- * happens to be false if bfqd is an NCQ-capable flash-based
- * device.
+ * above conditions (a), (b) or (c) is true, and, in
+ * particular, happens to be false if bfqd is an NCQ-capable
+ * flash-based device.
*/
- idling_boosts_thr = !bfqd->hw_tag ||
- (!blk_queue_nonrot(bfqd->queue) && bfq_bfqq_IO_bound(bfqq) &&
- bfq_bfqq_idle_window(bfqq));
+ idling_boosts_thr = rot_without_queueing ||
+ ((!blk_queue_nonrot(bfqd->queue) || !bfqd->hw_tag) &&
+ bfqq_sequential_and_IO_bound);
/*
* The value of the next variable,
asymmetric_scenario && !bfq_bfqq_in_large_burst(bfqq);
/*
- * We have now all the components we need to compute the return
- * value of the function, which is true only if both the following
- * conditions hold:
- * 1) bfqq is sync, because idling make sense only for sync queues;
- * 2) idling either boosts the throughput (without issues), or
- * is necessary to preserve service guarantees.
+ * We have now all the components we need to compute the
+ * return value of the function, which is true only if idling
+ * either boosts the throughput (without issues), or is
+ * necessary to preserve service guarantees.
*/
- return bfq_bfqq_sync(bfqq) &&
- (idling_boosts_thr_without_issues ||
- idling_needed_for_service_guarantees);
+ return idling_boosts_thr_without_issues ||
+ idling_needed_for_service_guarantees;
}
/*
*/
static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq)
{
- struct bfq_data *bfqd = bfqq->bfqd;
-
- return RB_EMPTY_ROOT(&bfqq->sort_list) && bfqd->bfq_slice_idle != 0 &&
- bfq_bfqq_may_idle(bfqq);
+ return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_may_idle(bfqq);
}
/*
default:
dev_err(bfqq->bfqd->queue->backing_dev_info->dev,
"bfq: bad prio class %d\n", ioprio_class);
+ /* fall through */
case IOPRIO_CLASS_NONE:
/*
* No prio set, inherit CPU scheduling settings.
case IOPRIO_CLASS_IDLE:
bfqq->new_ioprio_class = IOPRIO_CLASS_IDLE;
bfqq->new_ioprio = 7;
- bfq_clear_bfqq_idle_window(bfqq);
break;
}
bfq_set_next_ioprio_data(bfqq, bic);
if (is_sync) {
+ /*
+ * No need to mark as has_short_ttime if in
+ * idle_class, because no device idling is performed
+ * for queues in idle class
+ */
if (!bfq_class_idle(bfqq))
- bfq_mark_bfqq_idle_window(bfqq);
+ /* tentatively mark as has_short_ttime */
+ bfq_mark_bfqq_has_short_ttime(bfqq);
bfq_mark_bfqq_sync(bfqq);
bfq_mark_bfqq_just_created(bfqq);
} else
blk_rq_sectors(rq) < BFQQ_SECT_THR_NONROT);
}
-/*
- * Disable idle window if the process thinks too long or seeks so much that
- * it doesn't matter.
- */
-static void bfq_update_idle_window(struct bfq_data *bfqd,
- struct bfq_queue *bfqq,
- struct bfq_io_cq *bic)
+static void bfq_update_has_short_ttime(struct bfq_data *bfqd,
+ struct bfq_queue *bfqq,
+ struct bfq_io_cq *bic)
{
- int enable_idle;
+ bool has_short_ttime = true;
- /* Don't idle for async or idle io prio class. */
- if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq))
+ /*
+ * No need to update has_short_ttime if bfqq is async or in
+ * idle io prio class, or if bfq_slice_idle is zero, because
+ * no device idling is performed for bfqq in this case.
+ */
+ if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq) ||
+ bfqd->bfq_slice_idle == 0)
return;
/* Idle window just restored, statistics are meaningless. */
bfqd->bfq_wr_min_idle_time))
return;
- enable_idle = bfq_bfqq_idle_window(bfqq);
-
+ /* Think time is infinite if no process is linked to
+ * bfqq. Otherwise check average think time to
+ * decide whether to mark as has_short_ttime
+ */
if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||
- bfqd->bfq_slice_idle == 0 ||
- (bfqd->hw_tag && BFQQ_SEEKY(bfqq) &&
- bfqq->wr_coeff == 1))
- enable_idle = 0;
- else if (bfq_sample_valid(bfqq->ttime.ttime_samples)) {
- if (bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle &&
- bfqq->wr_coeff == 1)
- enable_idle = 0;
- else
- enable_idle = 1;
- }
- bfq_log_bfqq(bfqd, bfqq, "update_idle_window: enable_idle %d",
- enable_idle);
+ (bfq_sample_valid(bfqq->ttime.ttime_samples) &&
+ bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle))
+ has_short_ttime = false;
+
+ bfq_log_bfqq(bfqd, bfqq, "update_has_short_ttime: has_short_ttime %d",
+ has_short_ttime);
- if (enable_idle)
- bfq_mark_bfqq_idle_window(bfqq);
+ if (has_short_ttime)
+ bfq_mark_bfqq_has_short_ttime(bfqq);
else
- bfq_clear_bfqq_idle_window(bfqq);
+ bfq_clear_bfqq_has_short_ttime(bfqq);
}
/*
bfqq->meta_pending++;
bfq_update_io_thinktime(bfqd, bfqq);
+ bfq_update_has_short_ttime(bfqd, bfqq, bic);
bfq_update_io_seektime(bfqd, bfqq, rq);
- if (bfqq->entity.service > bfq_max_budget(bfqd) / 8 ||
- !BFQQ_SEEKY(bfqq))
- bfq_update_idle_window(bfqd, bfqq, bic);
bfq_log_bfqq(bfqd, bfqq,
- "rq_enqueued: idle_window=%d (seeky %d)",
- bfq_bfqq_idle_window(bfqq), BFQQ_SEEKY(bfqq));
+ "rq_enqueued: has_short_ttime=%d (seeky %d)",
+ bfq_bfqq_has_short_ttime(bfqq), BFQQ_SEEKY(bfqq));
bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
return sprintf(page, "%u\n", var);
}
-static ssize_t bfq_var_store(unsigned long *var, const char *page,
- size_t count)
+static void bfq_var_store(unsigned long *var, const char *page)
{
unsigned long new_val;
int ret = kstrtoul(page, 10, &new_val);
if (ret == 0)
*var = new_val;
-
- return count;
}
#define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \
{ \
struct bfq_data *bfqd = e->elevator_data; \
unsigned long uninitialized_var(__data); \
- int ret = bfq_var_store(&__data, (page), count); \
+ bfq_var_store(&__data, (page)); \
if (__data < (MIN)) \
__data = (MIN); \
else if (__data > (MAX)) \
*(__PTR) = (u64)__data * NSEC_PER_MSEC; \
else \
*(__PTR) = __data; \
- return ret; \
+ return count; \
}
STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1,
INT_MAX, 2);
{ \
struct bfq_data *bfqd = e->elevator_data; \
unsigned long uninitialized_var(__data); \
- int ret = bfq_var_store(&__data, (page), count); \
+ bfq_var_store(&__data, (page)); \
if (__data < (MIN)) \
__data = (MIN); \
else if (__data > (MAX)) \
__data = (MAX); \
*(__PTR) = (u64)__data * NSEC_PER_USEC; \
- return ret; \
+ return count; \
}
USEC_STORE_FUNCTION(bfq_slice_idle_us_store, &bfqd->bfq_slice_idle, 0,
UINT_MAX);
{
struct bfq_data *bfqd = e->elevator_data;
unsigned long uninitialized_var(__data);
- int ret = bfq_var_store(&__data, (page), count);
+
+ bfq_var_store(&__data, (page));
if (__data == 0)
bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd);
bfqd->bfq_user_max_budget = __data;
- return ret;
+ return count;
}
/*
{
struct bfq_data *bfqd = e->elevator_data;
unsigned long uninitialized_var(__data);
- int ret = bfq_var_store(&__data, (page), count);
+
+ bfq_var_store(&__data, (page));
if (__data < 1)
__data = 1;
if (bfqd->bfq_user_max_budget == 0)
bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd);
- return ret;
+ return count;
}
static ssize_t bfq_strict_guarantees_store(struct elevator_queue *e,
{
struct bfq_data *bfqd = e->elevator_data;
unsigned long uninitialized_var(__data);
- int ret = bfq_var_store(&__data, (page), count);
+
+ bfq_var_store(&__data, (page));
if (__data > 1)
__data = 1;
bfqd->strict_guarantees = __data;
- return ret;
+ return count;
}
static ssize_t bfq_low_latency_store(struct elevator_queue *e,
{
struct bfq_data *bfqd = e->elevator_data;
unsigned long uninitialized_var(__data);
- int ret = bfq_var_store(&__data, (page), count);
+
+ bfq_var_store(&__data, (page));
if (__data > 1)
__data = 1;
bfq_end_wr(bfqd);
bfqd->low_latency = __data;
- return ret;
+ return count;
}
#define BFQ_ATTR(name) \
ret = elv_register(&iosched_bfq_mq);
if (ret)
- goto err_pol_unreg;
+ goto slab_kill;
return 0;
+slab_kill:
+ bfq_slab_kill();
err_pol_unreg:
#ifdef CONFIG_BFQ_GROUP_IOSCHED
blkcg_policy_unregister(&blkcg_policy_bfq);
projects / mirror_ubuntu-bionic-kernel.git / blobdiff