]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4 LT |
2 | /* |
3 | * net/sched/gen_estimator.c Simple rate estimator. | |
4 | * | |
1da177e4 | 5 | * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> |
1c0d32fd | 6 | * Eric Dumazet <edumazet@google.com> |
1da177e4 LT |
7 | * |
8 | * Changes: | |
9 | * Jamal Hadi Salim - moved it to net/core and reshulfed | |
10 | * names to make it usable in general net subsystem. | |
11 | */ | |
12 | ||
7c0f6ba6 | 13 | #include <linux/uaccess.h> |
1977f032 | 14 | #include <linux/bitops.h> |
1da177e4 LT |
15 | #include <linux/module.h> |
16 | #include <linux/types.h> | |
17 | #include <linux/kernel.h> | |
18 | #include <linux/jiffies.h> | |
19 | #include <linux/string.h> | |
20 | #include <linux/mm.h> | |
21 | #include <linux/socket.h> | |
22 | #include <linux/sockios.h> | |
23 | #include <linux/in.h> | |
24 | #include <linux/errno.h> | |
25 | #include <linux/interrupt.h> | |
26 | #include <linux/netdevice.h> | |
27 | #include <linux/skbuff.h> | |
28 | #include <linux/rtnetlink.h> | |
29 | #include <linux/init.h> | |
5a0e3ad6 | 30 | #include <linux/slab.h> |
1c0d32fd | 31 | #include <linux/seqlock.h> |
1da177e4 LT |
32 | #include <net/sock.h> |
33 | #include <net/gen_stats.h> | |
34 | ||
1c0d32fd ED |
35 | /* This code is NOT intended to be used for statistics collection, |
36 | * its purpose is to provide a base for statistical multiplexing | |
37 | * for controlled load service. | |
38 | * If you need only statistics, run a user level daemon which | |
39 | * periodically reads byte counters. | |
1da177e4 LT |
40 | */ |
41 | ||
1c0d32fd | 42 | struct net_rate_estimator { |
c1a8f1f1 | 43 | struct gnet_stats_basic_packed *bstats; |
1da177e4 | 44 | spinlock_t *stats_lock; |
edb09eb1 | 45 | seqcount_t *running; |
1c0d32fd ED |
46 | struct gnet_stats_basic_cpu __percpu *cpu_bstats; |
47 | u8 ewma_log; | |
48 | u8 intvl_log; /* period : (250ms << intvl_log) */ | |
49 | ||
50 | seqcount_t seq; | |
1c8dd9cb | 51 | u64 last_packets; |
1da177e4 | 52 | u64 last_bytes; |
1c0d32fd ED |
53 | |
54 | u64 avpps; | |
511e11e3 | 55 | u64 avbps; |
1da177e4 | 56 | |
1c0d32fd ED |
57 | unsigned long next_jiffies; |
58 | struct timer_list timer; | |
59 | struct rcu_head rcu; | |
1da177e4 LT |
60 | }; |
61 | ||
1c0d32fd ED |
62 | static void est_fetch_counters(struct net_rate_estimator *e, |
63 | struct gnet_stats_basic_packed *b) | |
1da177e4 | 64 | { |
a5f7add3 | 65 | memset(b, 0, sizeof(*b)); |
1c0d32fd ED |
66 | if (e->stats_lock) |
67 | spin_lock(e->stats_lock); | |
1da177e4 | 68 | |
1c0d32fd | 69 | __gnet_stats_copy_basic(e->running, b, e->cpu_bstats, e->bstats); |
1da177e4 | 70 | |
1c0d32fd ED |
71 | if (e->stats_lock) |
72 | spin_unlock(e->stats_lock); | |
12efa1fa | 73 | |
1da177e4 LT |
74 | } |
75 | ||
e99e88a9 | 76 | static void est_timer(struct timer_list *t) |
4db0acf3 | 77 | { |
e99e88a9 | 78 | struct net_rate_estimator *est = from_timer(est, t, timer); |
1c0d32fd ED |
79 | struct gnet_stats_basic_packed b; |
80 | u64 rate, brate; | |
4db0acf3 | 81 | |
1c0d32fd | 82 | est_fetch_counters(est, &b); |
dd5e0733 ED |
83 | brate = (b.bytes - est->last_bytes) << (10 - est->intvl_log); |
84 | brate = (brate >> est->ewma_log) - (est->avbps >> est->ewma_log); | |
4db0acf3 | 85 | |
dd5e0733 ED |
86 | rate = (b.packets - est->last_packets) << (10 - est->intvl_log); |
87 | rate = (rate >> est->ewma_log) - (est->avpps >> est->ewma_log); | |
4db0acf3 | 88 | |
1c0d32fd ED |
89 | write_seqcount_begin(&est->seq); |
90 | est->avbps += brate; | |
91 | est->avpps += rate; | |
92 | write_seqcount_end(&est->seq); | |
4db0acf3 | 93 | |
1c0d32fd ED |
94 | est->last_bytes = b.bytes; |
95 | est->last_packets = b.packets; | |
4db0acf3 | 96 | |
1c0d32fd | 97 | est->next_jiffies += ((HZ/4) << est->intvl_log); |
4db0acf3 | 98 | |
1c0d32fd ED |
99 | if (unlikely(time_after_eq(jiffies, est->next_jiffies))) { |
100 | /* Ouch... timer was delayed. */ | |
101 | est->next_jiffies = jiffies + 1; | |
4db0acf3 | 102 | } |
1c0d32fd | 103 | mod_timer(&est->timer, est->next_jiffies); |
4db0acf3 JP |
104 | } |
105 | ||
1da177e4 LT |
106 | /** |
107 | * gen_new_estimator - create a new rate estimator | |
108 | * @bstats: basic statistics | |
e9fc2f05 | 109 | * @cpu_bstats: bstats per cpu |
1da177e4 | 110 | * @rate_est: rate estimator statistics |
51a9f5ae | 111 | * @lock: lock for statistics and control path |
edb09eb1 | 112 | * @running: qdisc running seqcount |
1da177e4 LT |
113 | * @opt: rate estimator configuration TLV |
114 | * | |
115 | * Creates a new rate estimator with &bstats as source and &rate_est | |
116 | * as destination. A new timer with the interval specified in the | |
117 | * configuration TLV is created. Upon each interval, the latest statistics | |
118 | * will be read from &bstats and the estimated rate will be stored in | |
e793c0f7 | 119 | * &rate_est with the statistics lock grabbed during this period. |
4ec93edb | 120 | * |
1da177e4 | 121 | * Returns 0 on success or a negative error code. |
0929c2dd | 122 | * |
1da177e4 | 123 | */ |
c1a8f1f1 | 124 | int gen_new_estimator(struct gnet_stats_basic_packed *bstats, |
22e0f8b9 | 125 | struct gnet_stats_basic_cpu __percpu *cpu_bstats, |
1c0d32fd | 126 | struct net_rate_estimator __rcu **rate_est, |
51a9f5ae | 127 | spinlock_t *lock, |
edb09eb1 | 128 | seqcount_t *running, |
1e90474c | 129 | struct nlattr *opt) |
1da177e4 | 130 | { |
1e90474c | 131 | struct gnet_estimator *parm = nla_data(opt); |
1c0d32fd ED |
132 | struct net_rate_estimator *old, *est; |
133 | struct gnet_stats_basic_packed b; | |
134 | int intvl_log; | |
1da177e4 | 135 | |
1e90474c | 136 | if (nla_len(opt) < sizeof(*parm)) |
1da177e4 LT |
137 | return -EINVAL; |
138 | ||
1c0d32fd ED |
139 | /* allowed timer periods are : |
140 | * -2 : 250ms, -1 : 500ms, 0 : 1 sec | |
141 | * 1 : 2 sec, 2 : 4 sec, 3 : 8 sec | |
142 | */ | |
1da177e4 LT |
143 | if (parm->interval < -2 || parm->interval > 3) |
144 | return -EINVAL; | |
145 | ||
dd5e0733 ED |
146 | if (parm->ewma_log == 0 || parm->ewma_log >= 31) |
147 | return -EINVAL; | |
148 | ||
77d04bd9 | 149 | est = kzalloc(sizeof(*est), GFP_KERNEL); |
1c0d32fd | 150 | if (!est) |
1da177e4 LT |
151 | return -ENOBUFS; |
152 | ||
1c0d32fd ED |
153 | seqcount_init(&est->seq); |
154 | intvl_log = parm->interval + 2; | |
1da177e4 | 155 | est->bstats = bstats; |
51a9f5ae | 156 | est->stats_lock = lock; |
edb09eb1 | 157 | est->running = running; |
1da177e4 | 158 | est->ewma_log = parm->ewma_log; |
1c0d32fd | 159 | est->intvl_log = intvl_log; |
22e0f8b9 | 160 | est->cpu_bstats = cpu_bstats; |
1da177e4 | 161 | |
51a9f5ae | 162 | if (lock) |
40ca54e3 | 163 | local_bh_disable(); |
1c0d32fd | 164 | est_fetch_counters(est, &b); |
51a9f5ae | 165 | if (lock) |
40ca54e3 | 166 | local_bh_enable(); |
1c0d32fd ED |
167 | est->last_bytes = b.bytes; |
168 | est->last_packets = b.packets; | |
51a9f5ae VB |
169 | |
170 | if (lock) | |
171 | spin_lock_bh(lock); | |
1c0d32fd ED |
172 | old = rcu_dereference_protected(*rate_est, 1); |
173 | if (old) { | |
174 | del_timer_sync(&old->timer); | |
175 | est->avbps = old->avbps; | |
176 | est->avpps = old->avpps; | |
1da177e4 | 177 | } |
0929c2dd | 178 | |
1c0d32fd | 179 | est->next_jiffies = jiffies + ((HZ/4) << intvl_log); |
e99e88a9 | 180 | timer_setup(&est->timer, est_timer, 0); |
1c0d32fd | 181 | mod_timer(&est->timer, est->next_jiffies); |
4db0acf3 | 182 | |
1c0d32fd | 183 | rcu_assign_pointer(*rate_est, est); |
51a9f5ae VB |
184 | if (lock) |
185 | spin_unlock_bh(lock); | |
1c0d32fd ED |
186 | if (old) |
187 | kfree_rcu(old, rcu); | |
1da177e4 LT |
188 | return 0; |
189 | } | |
c1b56878 | 190 | EXPORT_SYMBOL(gen_new_estimator); |
1da177e4 LT |
191 | |
192 | /** | |
193 | * gen_kill_estimator - remove a rate estimator | |
1c0d32fd | 194 | * @rate_est: rate estimator |
1da177e4 | 195 | * |
1c0d32fd | 196 | * Removes the rate estimator. |
0929c2dd | 197 | * |
1da177e4 | 198 | */ |
1c0d32fd | 199 | void gen_kill_estimator(struct net_rate_estimator __rcu **rate_est) |
1da177e4 | 200 | { |
1c0d32fd | 201 | struct net_rate_estimator *est; |
1da177e4 | 202 | |
1c0d32fd ED |
203 | est = xchg((__force struct net_rate_estimator **)rate_est, NULL); |
204 | if (est) { | |
205 | del_timer_sync(&est->timer); | |
206 | kfree_rcu(est, rcu); | |
1da177e4 LT |
207 | } |
208 | } | |
c1b56878 | 209 | EXPORT_SYMBOL(gen_kill_estimator); |
1da177e4 LT |
210 | |
211 | /** | |
96750162 | 212 | * gen_replace_estimator - replace rate estimator configuration |
1da177e4 | 213 | * @bstats: basic statistics |
e9fc2f05 | 214 | * @cpu_bstats: bstats per cpu |
1da177e4 | 215 | * @rate_est: rate estimator statistics |
51a9f5ae | 216 | * @lock: lock for statistics and control path |
edb09eb1 | 217 | * @running: qdisc running seqcount (might be NULL) |
1da177e4 LT |
218 | * @opt: rate estimator configuration TLV |
219 | * | |
220 | * Replaces the configuration of a rate estimator by calling | |
221 | * gen_kill_estimator() and gen_new_estimator(). | |
4ec93edb | 222 | * |
1da177e4 LT |
223 | * Returns 0 on success or a negative error code. |
224 | */ | |
c1a8f1f1 | 225 | int gen_replace_estimator(struct gnet_stats_basic_packed *bstats, |
22e0f8b9 | 226 | struct gnet_stats_basic_cpu __percpu *cpu_bstats, |
1c0d32fd | 227 | struct net_rate_estimator __rcu **rate_est, |
51a9f5ae | 228 | spinlock_t *lock, |
edb09eb1 | 229 | seqcount_t *running, struct nlattr *opt) |
1da177e4 | 230 | { |
1c0d32fd | 231 | return gen_new_estimator(bstats, cpu_bstats, rate_est, |
51a9f5ae | 232 | lock, running, opt); |
1da177e4 | 233 | } |
c1b56878 SH |
234 | EXPORT_SYMBOL(gen_replace_estimator); |
235 | ||
236 | /** | |
237 | * gen_estimator_active - test if estimator is currently in use | |
1c0d32fd | 238 | * @rate_est: rate estimator |
c1b56878 | 239 | * |
244e6c2d | 240 | * Returns true if estimator is active, and false if not. |
c1b56878 | 241 | */ |
1c0d32fd | 242 | bool gen_estimator_active(struct net_rate_estimator __rcu **rate_est) |
c1b56878 | 243 | { |
1c0d32fd ED |
244 | return !!rcu_access_pointer(*rate_est); |
245 | } | |
246 | EXPORT_SYMBOL(gen_estimator_active); | |
ae638c47 | 247 | |
1c0d32fd ED |
248 | bool gen_estimator_read(struct net_rate_estimator __rcu **rate_est, |
249 | struct gnet_stats_rate_est64 *sample) | |
250 | { | |
251 | struct net_rate_estimator *est; | |
252 | unsigned seq; | |
253 | ||
254 | rcu_read_lock(); | |
255 | est = rcu_dereference(*rate_est); | |
256 | if (!est) { | |
257 | rcu_read_unlock(); | |
258 | return false; | |
259 | } | |
1da177e4 | 260 | |
1c0d32fd ED |
261 | do { |
262 | seq = read_seqcount_begin(&est->seq); | |
263 | sample->bps = est->avbps >> 8; | |
264 | sample->pps = est->avpps >> 8; | |
265 | } while (read_seqcount_retry(&est->seq, seq)); | |
ae638c47 | 266 | |
1c0d32fd ED |
267 | rcu_read_unlock(); |
268 | return true; | |
c1b56878 | 269 | } |
1c0d32fd | 270 | EXPORT_SYMBOL(gen_estimator_read); |