]>
Commit | Line | Data |
---|---|---|
2d116e3e DL |
1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ | |
3 | ||
4 | #include "eswitch.h" | |
5 | #include "esw/qos.h" | |
ad34f02f | 6 | #include "en/port.h" |
3202ea65 DL |
7 | #define CREATE_TRACE_POINTS |
8 | #include "diag/qos_tracepoint.h" | |
2d116e3e DL |
9 | |
10 | /* Minimum supported BW share value by the HW is 1 Mbit/sec */ | |
11 | #define MLX5_MIN_BW_SHARE 1 | |
12 | ||
13 | #define MLX5_RATE_TO_BW_SHARE(rate, divider, limit) \ | |
ad34f02f | 14 | min_t(u32, max_t(u32, DIV_ROUND_UP(rate, divider), MLX5_MIN_BW_SHARE), limit) |
2d116e3e | 15 | |
1ae258f8 DL |
16 | struct mlx5_esw_rate_group { |
17 | u32 tsar_ix; | |
18 | u32 max_rate; | |
19 | u32 min_rate; | |
20 | u32 bw_share; | |
f47e04eb | 21 | struct list_head list; |
1ae258f8 DL |
22 | }; |
23 | ||
f47e04eb DL |
24 | static int esw_qos_tsar_config(struct mlx5_core_dev *dev, u32 *sched_ctx, |
25 | u32 parent_ix, u32 tsar_ix, | |
26 | u32 max_rate, u32 bw_share) | |
27 | { | |
28 | u32 bitmask = 0; | |
29 | ||
30 | if (!MLX5_CAP_GEN(dev, qos) || !MLX5_CAP_QOS(dev, esw_scheduling)) | |
31 | return -EOPNOTSUPP; | |
32 | ||
33 | MLX5_SET(scheduling_context, sched_ctx, parent_element_id, parent_ix); | |
34 | MLX5_SET(scheduling_context, sched_ctx, max_average_bw, max_rate); | |
35 | MLX5_SET(scheduling_context, sched_ctx, bw_share, bw_share); | |
36 | bitmask |= MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW; | |
37 | bitmask |= MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_BW_SHARE; | |
38 | ||
39 | return mlx5_modify_scheduling_element_cmd(dev, | |
40 | SCHEDULING_HIERARCHY_E_SWITCH, | |
41 | sched_ctx, | |
42 | tsar_ix, | |
43 | bitmask); | |
44 | } | |
45 | ||
46 | static int esw_qos_group_config(struct mlx5_eswitch *esw, struct mlx5_esw_rate_group *group, | |
47 | u32 max_rate, u32 bw_share, struct netlink_ext_ack *extack) | |
48 | { | |
49 | u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; | |
50 | struct mlx5_core_dev *dev = esw->dev; | |
51 | int err; | |
52 | ||
53 | err = esw_qos_tsar_config(dev, sched_ctx, | |
54 | esw->qos.root_tsar_ix, group->tsar_ix, | |
55 | max_rate, bw_share); | |
56 | if (err) | |
57 | NL_SET_ERR_MSG_MOD(extack, "E-Switch modify group TSAR element failed"); | |
58 | ||
3202ea65 DL |
59 | trace_mlx5_esw_group_qos_config(dev, group, group->tsar_ix, bw_share, max_rate); |
60 | ||
f47e04eb DL |
61 | return err; |
62 | } | |
63 | ||
2d116e3e DL |
64 | static int esw_qos_vport_config(struct mlx5_eswitch *esw, |
65 | struct mlx5_vport *vport, | |
ad34f02f DL |
66 | u32 max_rate, u32 bw_share, |
67 | struct netlink_ext_ack *extack) | |
2d116e3e DL |
68 | { |
69 | u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; | |
0fe132ea | 70 | struct mlx5_esw_rate_group *group = vport->qos.group; |
2d116e3e | 71 | struct mlx5_core_dev *dev = esw->dev; |
0fe132ea | 72 | u32 parent_tsar_ix; |
2d116e3e | 73 | void *vport_elem; |
2d116e3e DL |
74 | int err; |
75 | ||
2d116e3e DL |
76 | if (!vport->qos.enabled) |
77 | return -EIO; | |
78 | ||
0fe132ea | 79 | parent_tsar_ix = group ? group->tsar_ix : esw->qos.root_tsar_ix; |
2d116e3e DL |
80 | MLX5_SET(scheduling_context, sched_ctx, element_type, |
81 | SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT); | |
82 | vport_elem = MLX5_ADDR_OF(scheduling_context, sched_ctx, | |
83 | element_attributes); | |
84 | MLX5_SET(vport_element, vport_elem, vport_number, vport->vport); | |
2d116e3e | 85 | |
0fe132ea | 86 | err = esw_qos_tsar_config(dev, sched_ctx, parent_tsar_ix, vport->qos.esw_tsar_ix, |
f47e04eb | 87 | max_rate, bw_share); |
2d116e3e | 88 | if (err) { |
f47e04eb DL |
89 | esw_warn(esw->dev, |
90 | "E-Switch modify TSAR vport element failed (vport=%d,err=%d)\n", | |
2d116e3e | 91 | vport->vport, err); |
ad34f02f | 92 | NL_SET_ERR_MSG_MOD(extack, "E-Switch modify TSAR vport element failed"); |
2d116e3e DL |
93 | return err; |
94 | } | |
95 | ||
3202ea65 DL |
96 | trace_mlx5_esw_vport_qos_config(vport, bw_share, max_rate); |
97 | ||
2d116e3e DL |
98 | return 0; |
99 | } | |
100 | ||
f47e04eb DL |
101 | static u32 esw_qos_calculate_min_rate_divider(struct mlx5_eswitch *esw, |
102 | struct mlx5_esw_rate_group *group, | |
103 | bool group_level) | |
2d116e3e DL |
104 | { |
105 | u32 fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share); | |
106 | struct mlx5_vport *evport; | |
107 | u32 max_guarantee = 0; | |
108 | unsigned long i; | |
109 | ||
f47e04eb DL |
110 | if (group_level) { |
111 | struct mlx5_esw_rate_group *group; | |
112 | ||
113 | list_for_each_entry(group, &esw->qos.groups, list) { | |
114 | if (group->min_rate < max_guarantee) | |
115 | continue; | |
116 | max_guarantee = group->min_rate; | |
117 | } | |
118 | } else { | |
119 | mlx5_esw_for_each_vport(esw, i, evport) { | |
120 | if (!evport->enabled || !evport->qos.enabled || | |
0fe132ea | 121 | evport->qos.group != group || evport->qos.min_rate < max_guarantee) |
f47e04eb DL |
122 | continue; |
123 | max_guarantee = evport->qos.min_rate; | |
124 | } | |
2d116e3e DL |
125 | } |
126 | ||
127 | if (max_guarantee) | |
128 | return max_t(u32, max_guarantee / fw_max_bw_share, 1); | |
0fe132ea DL |
129 | |
130 | /* If vports min rate divider is 0 but their group has bw_share configured, then | |
131 | * need to set bw_share for vports to minimal value. | |
132 | */ | |
10daaa93 | 133 | if (!group_level && !max_guarantee && group && group->bw_share) |
0fe132ea | 134 | return 1; |
2d116e3e DL |
135 | return 0; |
136 | } | |
137 | ||
f47e04eb DL |
138 | static u32 esw_qos_calc_bw_share(u32 min_rate, u32 divider, u32 fw_max) |
139 | { | |
140 | if (divider) | |
141 | return MLX5_RATE_TO_BW_SHARE(min_rate, divider, fw_max); | |
142 | ||
143 | return 0; | |
144 | } | |
145 | ||
146 | static int esw_qos_normalize_vports_min_rate(struct mlx5_eswitch *esw, | |
147 | struct mlx5_esw_rate_group *group, | |
148 | struct netlink_ext_ack *extack) | |
2d116e3e DL |
149 | { |
150 | u32 fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share); | |
f47e04eb | 151 | u32 divider = esw_qos_calculate_min_rate_divider(esw, group, false); |
2d116e3e | 152 | struct mlx5_vport *evport; |
2d116e3e DL |
153 | unsigned long i; |
154 | u32 bw_share; | |
155 | int err; | |
156 | ||
157 | mlx5_esw_for_each_vport(esw, i, evport) { | |
0fe132ea | 158 | if (!evport->enabled || !evport->qos.enabled || evport->qos.group != group) |
2d116e3e | 159 | continue; |
f47e04eb | 160 | bw_share = esw_qos_calc_bw_share(evport->qos.min_rate, divider, fw_max_bw_share); |
2d116e3e DL |
161 | |
162 | if (bw_share == evport->qos.bw_share) | |
163 | continue; | |
164 | ||
f47e04eb DL |
165 | err = esw_qos_vport_config(esw, evport, evport->qos.max_rate, bw_share, extack); |
166 | if (err) | |
167 | return err; | |
168 | ||
169 | evport->qos.bw_share = bw_share; | |
170 | } | |
171 | ||
172 | return 0; | |
173 | } | |
174 | ||
175 | static int esw_qos_normalize_groups_min_rate(struct mlx5_eswitch *esw, u32 divider, | |
176 | struct netlink_ext_ack *extack) | |
177 | { | |
178 | u32 fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share); | |
179 | struct mlx5_esw_rate_group *group; | |
180 | u32 bw_share; | |
181 | int err; | |
182 | ||
183 | list_for_each_entry(group, &esw->qos.groups, list) { | |
184 | bw_share = esw_qos_calc_bw_share(group->min_rate, divider, fw_max_bw_share); | |
185 | ||
186 | if (bw_share == group->bw_share) | |
187 | continue; | |
188 | ||
189 | err = esw_qos_group_config(esw, group, group->max_rate, bw_share, extack); | |
190 | if (err) | |
2d116e3e | 191 | return err; |
f47e04eb DL |
192 | |
193 | group->bw_share = bw_share; | |
0fe132ea DL |
194 | |
195 | /* All the group's vports need to be set with default bw_share | |
196 | * to enable them with QOS | |
197 | */ | |
198 | err = esw_qos_normalize_vports_min_rate(esw, group, extack); | |
199 | ||
200 | if (err) | |
201 | return err; | |
2d116e3e DL |
202 | } |
203 | ||
204 | return 0; | |
205 | } | |
206 | ||
ad34f02f DL |
207 | int mlx5_esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw, |
208 | struct mlx5_vport *evport, | |
209 | u32 min_rate, | |
210 | struct netlink_ext_ack *extack) | |
2d116e3e | 211 | { |
ad34f02f | 212 | u32 fw_max_bw_share, previous_min_rate; |
2d116e3e | 213 | bool min_rate_supported; |
2d116e3e DL |
214 | int err; |
215 | ||
ad34f02f | 216 | lockdep_assert_held(&esw->state_lock); |
2d116e3e DL |
217 | fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share); |
218 | min_rate_supported = MLX5_CAP_QOS(esw->dev, esw_bw_share) && | |
219 | fw_max_bw_share >= MLX5_MIN_BW_SHARE; | |
ad34f02f | 220 | if (min_rate && !min_rate_supported) |
2d116e3e | 221 | return -EOPNOTSUPP; |
2d116e3e | 222 | if (min_rate == evport->qos.min_rate) |
ad34f02f | 223 | return 0; |
2d116e3e DL |
224 | |
225 | previous_min_rate = evport->qos.min_rate; | |
226 | evport->qos.min_rate = min_rate; | |
0fe132ea | 227 | err = esw_qos_normalize_vports_min_rate(esw, evport->qos.group, extack); |
ad34f02f | 228 | if (err) |
2d116e3e | 229 | evport->qos.min_rate = previous_min_rate; |
2d116e3e | 230 | |
ad34f02f DL |
231 | return err; |
232 | } | |
233 | ||
234 | int mlx5_esw_qos_set_vport_max_rate(struct mlx5_eswitch *esw, | |
235 | struct mlx5_vport *evport, | |
236 | u32 max_rate, | |
237 | struct netlink_ext_ack *extack) | |
238 | { | |
0fe132ea | 239 | u32 act_max_rate = max_rate; |
ad34f02f DL |
240 | bool max_rate_supported; |
241 | int err; | |
242 | ||
243 | lockdep_assert_held(&esw->state_lock); | |
244 | max_rate_supported = MLX5_CAP_QOS(esw->dev, esw_rate_limit); | |
245 | ||
246 | if (max_rate && !max_rate_supported) | |
247 | return -EOPNOTSUPP; | |
2d116e3e DL |
248 | if (max_rate == evport->qos.max_rate) |
249 | return 0; | |
250 | ||
0fe132ea DL |
251 | /* If parent group has rate limit need to set to group |
252 | * value when new max rate is 0. | |
253 | */ | |
254 | if (evport->qos.group && !max_rate) | |
255 | act_max_rate = evport->qos.group->max_rate; | |
256 | ||
257 | err = esw_qos_vport_config(esw, evport, act_max_rate, evport->qos.bw_share, extack); | |
f47e04eb | 258 | |
2d116e3e DL |
259 | if (!err) |
260 | evport->qos.max_rate = max_rate; | |
261 | ||
262 | return err; | |
263 | } | |
264 | ||
f47e04eb DL |
265 | static int esw_qos_set_group_min_rate(struct mlx5_eswitch *esw, struct mlx5_esw_rate_group *group, |
266 | u32 min_rate, struct netlink_ext_ack *extack) | |
267 | { | |
268 | u32 fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share); | |
269 | struct mlx5_core_dev *dev = esw->dev; | |
270 | u32 previous_min_rate, divider; | |
271 | int err; | |
272 | ||
273 | if (!(MLX5_CAP_QOS(dev, esw_bw_share) && fw_max_bw_share >= MLX5_MIN_BW_SHARE)) | |
274 | return -EOPNOTSUPP; | |
275 | ||
276 | if (min_rate == group->min_rate) | |
277 | return 0; | |
278 | ||
279 | previous_min_rate = group->min_rate; | |
280 | group->min_rate = min_rate; | |
281 | divider = esw_qos_calculate_min_rate_divider(esw, group, true); | |
282 | err = esw_qos_normalize_groups_min_rate(esw, divider, extack); | |
283 | if (err) { | |
284 | group->min_rate = previous_min_rate; | |
285 | NL_SET_ERR_MSG_MOD(extack, "E-Switch group min rate setting failed"); | |
286 | ||
287 | /* Attempt restoring previous configuration */ | |
288 | divider = esw_qos_calculate_min_rate_divider(esw, group, true); | |
289 | if (esw_qos_normalize_groups_min_rate(esw, divider, extack)) | |
290 | NL_SET_ERR_MSG_MOD(extack, "E-Switch BW share restore failed"); | |
291 | } | |
292 | ||
293 | return err; | |
294 | } | |
295 | ||
296 | static int esw_qos_set_group_max_rate(struct mlx5_eswitch *esw, | |
297 | struct mlx5_esw_rate_group *group, | |
298 | u32 max_rate, struct netlink_ext_ack *extack) | |
299 | { | |
0fe132ea DL |
300 | struct mlx5_vport *vport; |
301 | unsigned long i; | |
f47e04eb DL |
302 | int err; |
303 | ||
304 | if (group->max_rate == max_rate) | |
305 | return 0; | |
306 | ||
307 | err = esw_qos_group_config(esw, group, max_rate, group->bw_share, extack); | |
308 | if (err) | |
309 | return err; | |
310 | ||
311 | group->max_rate = max_rate; | |
312 | ||
0fe132ea DL |
313 | /* Any unlimited vports in the group should be set |
314 | * with the value of the group. | |
315 | */ | |
316 | mlx5_esw_for_each_vport(esw, i, vport) { | |
317 | if (!vport->enabled || !vport->qos.enabled || | |
318 | vport->qos.group != group || vport->qos.max_rate) | |
319 | continue; | |
320 | ||
321 | err = esw_qos_vport_config(esw, vport, max_rate, vport->qos.bw_share, extack); | |
322 | if (err) | |
323 | NL_SET_ERR_MSG_MOD(extack, | |
324 | "E-Switch vport implicit rate limit setting failed"); | |
325 | } | |
326 | ||
327 | return err; | |
328 | } | |
329 | ||
330 | static int esw_qos_vport_create_sched_element(struct mlx5_eswitch *esw, | |
331 | struct mlx5_vport *vport, | |
332 | u32 max_rate, u32 bw_share) | |
333 | { | |
334 | u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; | |
335 | struct mlx5_esw_rate_group *group = vport->qos.group; | |
336 | struct mlx5_core_dev *dev = esw->dev; | |
337 | u32 parent_tsar_ix; | |
338 | void *vport_elem; | |
339 | int err; | |
340 | ||
341 | parent_tsar_ix = group ? group->tsar_ix : esw->qos.root_tsar_ix; | |
342 | MLX5_SET(scheduling_context, sched_ctx, element_type, | |
343 | SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT); | |
344 | vport_elem = MLX5_ADDR_OF(scheduling_context, sched_ctx, element_attributes); | |
345 | MLX5_SET(vport_element, vport_elem, vport_number, vport->vport); | |
346 | MLX5_SET(scheduling_context, sched_ctx, parent_element_id, parent_tsar_ix); | |
347 | MLX5_SET(scheduling_context, sched_ctx, max_average_bw, max_rate); | |
348 | MLX5_SET(scheduling_context, sched_ctx, bw_share, bw_share); | |
349 | ||
350 | err = mlx5_create_scheduling_element_cmd(dev, | |
351 | SCHEDULING_HIERARCHY_E_SWITCH, | |
352 | sched_ctx, | |
353 | &vport->qos.esw_tsar_ix); | |
354 | if (err) { | |
355 | esw_warn(esw->dev, "E-Switch create TSAR vport element failed (vport=%d,err=%d)\n", | |
356 | vport->vport, err); | |
357 | return err; | |
358 | } | |
359 | ||
360 | return 0; | |
361 | } | |
362 | ||
363 | static int esw_qos_update_group_scheduling_element(struct mlx5_eswitch *esw, | |
364 | struct mlx5_vport *vport, | |
365 | struct mlx5_esw_rate_group *curr_group, | |
366 | struct mlx5_esw_rate_group *new_group, | |
367 | struct netlink_ext_ack *extack) | |
368 | { | |
369 | u32 max_rate; | |
370 | int err; | |
371 | ||
372 | err = mlx5_destroy_scheduling_element_cmd(esw->dev, | |
373 | SCHEDULING_HIERARCHY_E_SWITCH, | |
374 | vport->qos.esw_tsar_ix); | |
375 | if (err) { | |
376 | NL_SET_ERR_MSG_MOD(extack, "E-Switch destroy TSAR vport element failed"); | |
377 | return err; | |
378 | } | |
379 | ||
380 | vport->qos.group = new_group; | |
381 | max_rate = vport->qos.max_rate ? vport->qos.max_rate : new_group->max_rate; | |
382 | ||
383 | /* If vport is unlimited, we set the group's value. | |
384 | * Therefore, if the group is limited it will apply to | |
385 | * the vport as well and if not, vport will remain unlimited. | |
386 | */ | |
387 | err = esw_qos_vport_create_sched_element(esw, vport, max_rate, vport->qos.bw_share); | |
388 | if (err) { | |
389 | NL_SET_ERR_MSG_MOD(extack, "E-Switch vport group set failed."); | |
390 | goto err_sched; | |
391 | } | |
392 | ||
393 | return 0; | |
394 | ||
395 | err_sched: | |
396 | vport->qos.group = curr_group; | |
397 | max_rate = vport->qos.max_rate ? vport->qos.max_rate : curr_group->max_rate; | |
398 | if (esw_qos_vport_create_sched_element(esw, vport, max_rate, vport->qos.bw_share)) | |
399 | esw_warn(esw->dev, "E-Switch vport group restore failed (vport=%d)\n", | |
400 | vport->vport); | |
401 | ||
f47e04eb DL |
402 | return err; |
403 | } | |
404 | ||
0fe132ea DL |
405 | static int esw_qos_vport_update_group(struct mlx5_eswitch *esw, |
406 | struct mlx5_vport *vport, | |
407 | struct mlx5_esw_rate_group *group, | |
408 | struct netlink_ext_ack *extack) | |
409 | { | |
410 | struct mlx5_esw_rate_group *new_group, *curr_group; | |
411 | int err; | |
412 | ||
413 | if (!vport->enabled) | |
414 | return -EINVAL; | |
415 | ||
416 | curr_group = vport->qos.group; | |
417 | new_group = group ?: esw->qos.group0; | |
418 | if (curr_group == new_group) | |
419 | return 0; | |
420 | ||
421 | err = esw_qos_update_group_scheduling_element(esw, vport, curr_group, new_group, extack); | |
422 | if (err) | |
423 | return err; | |
424 | ||
425 | /* Recalculate bw share weights of old and new groups */ | |
4ac7e82c | 426 | if (vport->qos.bw_share || new_group->bw_share) { |
0fe132ea DL |
427 | esw_qos_normalize_vports_min_rate(esw, curr_group, extack); |
428 | esw_qos_normalize_vports_min_rate(esw, new_group, extack); | |
429 | } | |
430 | ||
431 | return 0; | |
432 | } | |
433 | ||
1ae258f8 DL |
434 | static struct mlx5_esw_rate_group * |
435 | esw_qos_create_rate_group(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) | |
436 | { | |
437 | u32 tsar_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; | |
438 | struct mlx5_esw_rate_group *group; | |
f47e04eb | 439 | u32 divider; |
1ae258f8 DL |
440 | int err; |
441 | ||
442 | if (!MLX5_CAP_QOS(esw->dev, log_esw_max_sched_depth)) | |
443 | return ERR_PTR(-EOPNOTSUPP); | |
444 | ||
445 | group = kzalloc(sizeof(*group), GFP_KERNEL); | |
446 | if (!group) | |
447 | return ERR_PTR(-ENOMEM); | |
448 | ||
449 | MLX5_SET(scheduling_context, tsar_ctx, parent_element_id, | |
450 | esw->qos.root_tsar_ix); | |
451 | err = mlx5_create_scheduling_element_cmd(esw->dev, | |
452 | SCHEDULING_HIERARCHY_E_SWITCH, | |
453 | tsar_ctx, | |
454 | &group->tsar_ix); | |
455 | if (err) { | |
456 | NL_SET_ERR_MSG_MOD(extack, "E-Switch create TSAR for group failed"); | |
457 | goto err_sched_elem; | |
458 | } | |
459 | ||
f47e04eb DL |
460 | list_add_tail(&group->list, &esw->qos.groups); |
461 | ||
462 | divider = esw_qos_calculate_min_rate_divider(esw, group, true); | |
463 | if (divider) { | |
464 | err = esw_qos_normalize_groups_min_rate(esw, divider, extack); | |
465 | if (err) { | |
466 | NL_SET_ERR_MSG_MOD(extack, "E-Switch groups normalization failed"); | |
467 | goto err_min_rate; | |
468 | } | |
469 | } | |
3202ea65 | 470 | trace_mlx5_esw_group_qos_create(esw->dev, group, group->tsar_ix); |
f47e04eb | 471 | |
1ae258f8 DL |
472 | return group; |
473 | ||
f47e04eb DL |
474 | err_min_rate: |
475 | list_del(&group->list); | |
a6f74333 DL |
476 | if (mlx5_destroy_scheduling_element_cmd(esw->dev, |
477 | SCHEDULING_HIERARCHY_E_SWITCH, | |
478 | group->tsar_ix)) | |
f47e04eb | 479 | NL_SET_ERR_MSG_MOD(extack, "E-Switch destroy TSAR for group failed"); |
1ae258f8 DL |
480 | err_sched_elem: |
481 | kfree(group); | |
482 | return ERR_PTR(err); | |
483 | } | |
484 | ||
485 | static int esw_qos_destroy_rate_group(struct mlx5_eswitch *esw, | |
486 | struct mlx5_esw_rate_group *group, | |
487 | struct netlink_ext_ack *extack) | |
488 | { | |
f47e04eb | 489 | u32 divider; |
1ae258f8 DL |
490 | int err; |
491 | ||
f47e04eb DL |
492 | list_del(&group->list); |
493 | ||
494 | divider = esw_qos_calculate_min_rate_divider(esw, NULL, true); | |
495 | err = esw_qos_normalize_groups_min_rate(esw, divider, extack); | |
496 | if (err) | |
497 | NL_SET_ERR_MSG_MOD(extack, "E-Switch groups' normalization failed"); | |
498 | ||
1ae258f8 DL |
499 | err = mlx5_destroy_scheduling_element_cmd(esw->dev, |
500 | SCHEDULING_HIERARCHY_E_SWITCH, | |
501 | group->tsar_ix); | |
502 | if (err) | |
503 | NL_SET_ERR_MSG_MOD(extack, "E-Switch destroy TSAR_ID failed"); | |
504 | ||
3202ea65 | 505 | trace_mlx5_esw_group_qos_destroy(esw->dev, group, group->tsar_ix); |
1ae258f8 DL |
506 | kfree(group); |
507 | return err; | |
508 | } | |
509 | ||
2d116e3e DL |
510 | static bool esw_qos_element_type_supported(struct mlx5_core_dev *dev, int type) |
511 | { | |
512 | switch (type) { | |
513 | case SCHEDULING_CONTEXT_ELEMENT_TYPE_TSAR: | |
514 | return MLX5_CAP_QOS(dev, esw_element_type) & | |
515 | ELEMENT_TYPE_CAP_MASK_TASR; | |
516 | case SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT: | |
517 | return MLX5_CAP_QOS(dev, esw_element_type) & | |
518 | ELEMENT_TYPE_CAP_MASK_VPORT; | |
519 | case SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT_TC: | |
520 | return MLX5_CAP_QOS(dev, esw_element_type) & | |
521 | ELEMENT_TYPE_CAP_MASK_VPORT_TC; | |
522 | case SCHEDULING_CONTEXT_ELEMENT_TYPE_PARA_VPORT_TC: | |
523 | return MLX5_CAP_QOS(dev, esw_element_type) & | |
524 | ELEMENT_TYPE_CAP_MASK_PARA_VPORT_TC; | |
525 | } | |
526 | return false; | |
527 | } | |
528 | ||
529 | void mlx5_esw_qos_create(struct mlx5_eswitch *esw) | |
530 | { | |
531 | u32 tsar_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; | |
532 | struct mlx5_core_dev *dev = esw->dev; | |
533 | __be32 *attr; | |
534 | int err; | |
535 | ||
536 | if (!MLX5_CAP_GEN(dev, qos) || !MLX5_CAP_QOS(dev, esw_scheduling)) | |
537 | return; | |
538 | ||
539 | if (!esw_qos_element_type_supported(dev, SCHEDULING_CONTEXT_ELEMENT_TYPE_TSAR)) | |
540 | return; | |
541 | ||
1ae258f8 | 542 | mutex_lock(&esw->state_lock); |
2d116e3e | 543 | if (esw->qos.enabled) |
1ae258f8 | 544 | goto unlock; |
2d116e3e DL |
545 | |
546 | MLX5_SET(scheduling_context, tsar_ctx, element_type, | |
547 | SCHEDULING_CONTEXT_ELEMENT_TYPE_TSAR); | |
548 | ||
549 | attr = MLX5_ADDR_OF(scheduling_context, tsar_ctx, element_attributes); | |
550 | *attr = cpu_to_be32(TSAR_ELEMENT_TSAR_TYPE_DWRR << 16); | |
551 | ||
552 | err = mlx5_create_scheduling_element_cmd(dev, | |
553 | SCHEDULING_HIERARCHY_E_SWITCH, | |
554 | tsar_ctx, | |
555 | &esw->qos.root_tsar_ix); | |
556 | if (err) { | |
1ae258f8 DL |
557 | esw_warn(dev, "E-Switch create root TSAR failed (%d)\n", err); |
558 | goto unlock; | |
2d116e3e DL |
559 | } |
560 | ||
f47e04eb | 561 | INIT_LIST_HEAD(&esw->qos.groups); |
1ae258f8 DL |
562 | if (MLX5_CAP_QOS(dev, log_esw_max_sched_depth)) { |
563 | esw->qos.group0 = esw_qos_create_rate_group(esw, NULL); | |
564 | if (IS_ERR(esw->qos.group0)) { | |
565 | esw_warn(dev, "E-Switch create rate group 0 failed (%ld)\n", | |
566 | PTR_ERR(esw->qos.group0)); | |
567 | goto err_group0; | |
568 | } | |
569 | } | |
2d116e3e | 570 | esw->qos.enabled = true; |
1ae258f8 DL |
571 | unlock: |
572 | mutex_unlock(&esw->state_lock); | |
573 | return; | |
574 | ||
575 | err_group0: | |
576 | err = mlx5_destroy_scheduling_element_cmd(esw->dev, | |
577 | SCHEDULING_HIERARCHY_E_SWITCH, | |
578 | esw->qos.root_tsar_ix); | |
579 | if (err) | |
580 | esw_warn(esw->dev, "E-Switch destroy root TSAR failed (%d)\n", err); | |
581 | mutex_unlock(&esw->state_lock); | |
2d116e3e DL |
582 | } |
583 | ||
584 | void mlx5_esw_qos_destroy(struct mlx5_eswitch *esw) | |
585 | { | |
1ae258f8 | 586 | struct devlink *devlink = priv_to_devlink(esw->dev); |
2d116e3e DL |
587 | int err; |
588 | ||
1ae258f8 DL |
589 | devlink_rate_nodes_destroy(devlink); |
590 | mutex_lock(&esw->state_lock); | |
2d116e3e | 591 | if (!esw->qos.enabled) |
1ae258f8 DL |
592 | goto unlock; |
593 | ||
594 | if (esw->qos.group0) | |
595 | esw_qos_destroy_rate_group(esw, esw->qos.group0, NULL); | |
2d116e3e DL |
596 | |
597 | err = mlx5_destroy_scheduling_element_cmd(esw->dev, | |
598 | SCHEDULING_HIERARCHY_E_SWITCH, | |
599 | esw->qos.root_tsar_ix); | |
600 | if (err) | |
1ae258f8 | 601 | esw_warn(esw->dev, "E-Switch destroy root TSAR failed (%d)\n", err); |
2d116e3e DL |
602 | |
603 | esw->qos.enabled = false; | |
1ae258f8 DL |
604 | unlock: |
605 | mutex_unlock(&esw->state_lock); | |
2d116e3e DL |
606 | } |
607 | ||
608 | int mlx5_esw_qos_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport, | |
609 | u32 max_rate, u32 bw_share) | |
610 | { | |
2d116e3e DL |
611 | int err; |
612 | ||
613 | lockdep_assert_held(&esw->state_lock); | |
614 | if (!esw->qos.enabled) | |
615 | return 0; | |
616 | ||
617 | if (vport->qos.enabled) | |
618 | return -EEXIST; | |
619 | ||
0fe132ea | 620 | vport->qos.group = esw->qos.group0; |
2d116e3e | 621 | |
0fe132ea | 622 | err = esw_qos_vport_create_sched_element(esw, vport, max_rate, bw_share); |
3202ea65 | 623 | if (!err) { |
2d116e3e | 624 | vport->qos.enabled = true; |
3202ea65 DL |
625 | trace_mlx5_esw_vport_qos_create(vport, bw_share, max_rate); |
626 | } | |
2d116e3e DL |
627 | |
628 | return err; | |
629 | } | |
630 | ||
631 | void mlx5_esw_qos_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vport) | |
632 | { | |
633 | int err; | |
634 | ||
635 | lockdep_assert_held(&esw->state_lock); | |
636 | if (!esw->qos.enabled || !vport->qos.enabled) | |
637 | return; | |
0fe132ea DL |
638 | WARN(vport->qos.group && vport->qos.group != esw->qos.group0, |
639 | "Disabling QoS on port before detaching it from group"); | |
2d116e3e DL |
640 | |
641 | err = mlx5_destroy_scheduling_element_cmd(esw->dev, | |
642 | SCHEDULING_HIERARCHY_E_SWITCH, | |
643 | vport->qos.esw_tsar_ix); | |
644 | if (err) | |
645 | esw_warn(esw->dev, "E-Switch destroy TSAR vport element failed (vport=%d,err=%d)\n", | |
646 | vport->vport, err); | |
647 | ||
648 | vport->qos.enabled = false; | |
3202ea65 | 649 | trace_mlx5_esw_vport_qos_destroy(vport); |
2d116e3e DL |
650 | } |
651 | ||
652 | int mlx5_esw_qos_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, u32 rate_mbps) | |
653 | { | |
654 | u32 ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; | |
655 | struct mlx5_vport *vport; | |
656 | u32 bitmask; | |
657 | ||
658 | vport = mlx5_eswitch_get_vport(esw, vport_num); | |
659 | if (IS_ERR(vport)) | |
660 | return PTR_ERR(vport); | |
661 | ||
662 | if (!vport->qos.enabled) | |
663 | return -EOPNOTSUPP; | |
664 | ||
665 | MLX5_SET(scheduling_context, ctx, max_average_bw, rate_mbps); | |
666 | bitmask = MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW; | |
667 | ||
668 | return mlx5_modify_scheduling_element_cmd(esw->dev, | |
669 | SCHEDULING_HIERARCHY_E_SWITCH, | |
670 | ctx, | |
671 | vport->qos.esw_tsar_ix, | |
672 | bitmask); | |
673 | } | |
ad34f02f DL |
674 | |
675 | #define MLX5_LINKSPEED_UNIT 125000 /* 1Mbps in Bps */ | |
676 | ||
677 | /* Converts bytes per second value passed in a pointer into megabits per | |
678 | * second, rewriting last. If converted rate exceed link speed or is not a | |
679 | * fraction of Mbps - returns error. | |
680 | */ | |
681 | static int esw_qos_devlink_rate_to_mbps(struct mlx5_core_dev *mdev, const char *name, | |
682 | u64 *rate, struct netlink_ext_ack *extack) | |
683 | { | |
684 | u32 link_speed_max, reminder; | |
685 | u64 value; | |
686 | int err; | |
687 | ||
688 | err = mlx5e_port_max_linkspeed(mdev, &link_speed_max); | |
689 | if (err) { | |
690 | NL_SET_ERR_MSG_MOD(extack, "Failed to get link maximum speed"); | |
691 | return err; | |
692 | } | |
693 | ||
694 | value = div_u64_rem(*rate, MLX5_LINKSPEED_UNIT, &reminder); | |
695 | if (reminder) { | |
696 | pr_err("%s rate value %lluBps not in link speed units of 1Mbps.\n", | |
697 | name, *rate); | |
698 | NL_SET_ERR_MSG_MOD(extack, "TX rate value not in link speed units of 1Mbps"); | |
699 | return -EINVAL; | |
700 | } | |
701 | ||
702 | if (value > link_speed_max) { | |
703 | pr_err("%s rate value %lluMbps exceed link maximum speed %u.\n", | |
704 | name, value, link_speed_max); | |
705 | NL_SET_ERR_MSG_MOD(extack, "TX rate value exceed link maximum speed"); | |
706 | return -EINVAL; | |
707 | } | |
708 | ||
709 | *rate = value; | |
710 | return 0; | |
711 | } | |
712 | ||
713 | /* Eswitch devlink rate API */ | |
714 | ||
715 | int mlx5_esw_devlink_rate_leaf_tx_share_set(struct devlink_rate *rate_leaf, void *priv, | |
716 | u64 tx_share, struct netlink_ext_ack *extack) | |
717 | { | |
718 | struct mlx5_vport *vport = priv; | |
719 | struct mlx5_eswitch *esw; | |
720 | int err; | |
721 | ||
722 | esw = vport->dev->priv.eswitch; | |
723 | if (!mlx5_esw_allowed(esw)) | |
724 | return -EPERM; | |
725 | ||
726 | err = esw_qos_devlink_rate_to_mbps(vport->dev, "tx_share", &tx_share, extack); | |
727 | if (err) | |
728 | return err; | |
729 | ||
730 | mutex_lock(&esw->state_lock); | |
731 | err = mlx5_esw_qos_set_vport_min_rate(esw, vport, tx_share, extack); | |
732 | mutex_unlock(&esw->state_lock); | |
733 | return err; | |
734 | } | |
735 | ||
736 | int mlx5_esw_devlink_rate_leaf_tx_max_set(struct devlink_rate *rate_leaf, void *priv, | |
737 | u64 tx_max, struct netlink_ext_ack *extack) | |
738 | { | |
739 | struct mlx5_vport *vport = priv; | |
740 | struct mlx5_eswitch *esw; | |
741 | int err; | |
742 | ||
743 | esw = vport->dev->priv.eswitch; | |
744 | if (!mlx5_esw_allowed(esw)) | |
745 | return -EPERM; | |
746 | ||
747 | err = esw_qos_devlink_rate_to_mbps(vport->dev, "tx_max", &tx_max, extack); | |
748 | if (err) | |
749 | return err; | |
750 | ||
751 | mutex_lock(&esw->state_lock); | |
752 | err = mlx5_esw_qos_set_vport_max_rate(esw, vport, tx_max, extack); | |
753 | mutex_unlock(&esw->state_lock); | |
754 | return err; | |
755 | } | |
1ae258f8 | 756 | |
f47e04eb DL |
757 | int mlx5_esw_devlink_rate_node_tx_share_set(struct devlink_rate *rate_node, void *priv, |
758 | u64 tx_share, struct netlink_ext_ack *extack) | |
759 | { | |
760 | struct mlx5_core_dev *dev = devlink_priv(rate_node->devlink); | |
761 | struct mlx5_eswitch *esw = dev->priv.eswitch; | |
762 | struct mlx5_esw_rate_group *group = priv; | |
763 | int err; | |
764 | ||
765 | err = esw_qos_devlink_rate_to_mbps(dev, "tx_share", &tx_share, extack); | |
766 | if (err) | |
767 | return err; | |
768 | ||
769 | mutex_lock(&esw->state_lock); | |
770 | err = esw_qos_set_group_min_rate(esw, group, tx_share, extack); | |
771 | mutex_unlock(&esw->state_lock); | |
772 | return err; | |
773 | } | |
774 | ||
775 | int mlx5_esw_devlink_rate_node_tx_max_set(struct devlink_rate *rate_node, void *priv, | |
776 | u64 tx_max, struct netlink_ext_ack *extack) | |
777 | { | |
778 | struct mlx5_core_dev *dev = devlink_priv(rate_node->devlink); | |
779 | struct mlx5_eswitch *esw = dev->priv.eswitch; | |
780 | struct mlx5_esw_rate_group *group = priv; | |
781 | int err; | |
782 | ||
783 | err = esw_qos_devlink_rate_to_mbps(dev, "tx_max", &tx_max, extack); | |
784 | if (err) | |
785 | return err; | |
786 | ||
787 | mutex_lock(&esw->state_lock); | |
788 | err = esw_qos_set_group_max_rate(esw, group, tx_max, extack); | |
789 | mutex_unlock(&esw->state_lock); | |
790 | return err; | |
791 | } | |
792 | ||
1ae258f8 DL |
793 | int mlx5_esw_devlink_rate_node_new(struct devlink_rate *rate_node, void **priv, |
794 | struct netlink_ext_ack *extack) | |
795 | { | |
796 | struct mlx5_esw_rate_group *group; | |
797 | struct mlx5_eswitch *esw; | |
798 | int err = 0; | |
799 | ||
800 | esw = mlx5_devlink_eswitch_get(rate_node->devlink); | |
801 | if (IS_ERR(esw)) | |
802 | return PTR_ERR(esw); | |
803 | ||
804 | mutex_lock(&esw->state_lock); | |
805 | if (esw->mode != MLX5_ESWITCH_OFFLOADS) { | |
806 | NL_SET_ERR_MSG_MOD(extack, | |
807 | "Rate node creation supported only in switchdev mode"); | |
808 | err = -EOPNOTSUPP; | |
809 | goto unlock; | |
810 | } | |
811 | ||
812 | group = esw_qos_create_rate_group(esw, extack); | |
813 | if (IS_ERR(group)) { | |
814 | err = PTR_ERR(group); | |
815 | goto unlock; | |
816 | } | |
817 | ||
818 | *priv = group; | |
819 | unlock: | |
820 | mutex_unlock(&esw->state_lock); | |
821 | return err; | |
822 | } | |
823 | ||
824 | int mlx5_esw_devlink_rate_node_del(struct devlink_rate *rate_node, void *priv, | |
825 | struct netlink_ext_ack *extack) | |
826 | { | |
827 | struct mlx5_esw_rate_group *group = priv; | |
828 | struct mlx5_eswitch *esw; | |
829 | int err; | |
830 | ||
831 | esw = mlx5_devlink_eswitch_get(rate_node->devlink); | |
832 | if (IS_ERR(esw)) | |
833 | return PTR_ERR(esw); | |
834 | ||
835 | mutex_lock(&esw->state_lock); | |
836 | err = esw_qos_destroy_rate_group(esw, group, extack); | |
837 | mutex_unlock(&esw->state_lock); | |
838 | return err; | |
839 | } | |
0fe132ea DL |
840 | |
841 | int mlx5_esw_qos_vport_update_group(struct mlx5_eswitch *esw, | |
842 | struct mlx5_vport *vport, | |
843 | struct mlx5_esw_rate_group *group, | |
844 | struct netlink_ext_ack *extack) | |
845 | { | |
846 | int err; | |
847 | ||
848 | mutex_lock(&esw->state_lock); | |
849 | err = esw_qos_vport_update_group(esw, vport, group, extack); | |
850 | mutex_unlock(&esw->state_lock); | |
851 | return err; | |
852 | } | |
853 | ||
854 | int mlx5_esw_devlink_rate_parent_set(struct devlink_rate *devlink_rate, | |
855 | struct devlink_rate *parent, | |
856 | void *priv, void *parent_priv, | |
857 | struct netlink_ext_ack *extack) | |
858 | { | |
859 | struct mlx5_esw_rate_group *group; | |
860 | struct mlx5_vport *vport = priv; | |
861 | ||
862 | if (!parent) | |
863 | return mlx5_esw_qos_vport_update_group(vport->dev->priv.eswitch, | |
864 | vport, NULL, extack); | |
865 | ||
866 | group = parent_priv; | |
867 | return mlx5_esw_qos_vport_update_group(vport->dev->priv.eswitch, vport, group, extack); | |
868 | } |