1 // SPDX-License-Identifier: GPL-2.0-only
3 * Pluggable TCP congestion control support and newReno
5 * Based on ideas from I/O scheduler support and Web100.
7 * Copyright (C) 2005 Stephen Hemminger <shemminger@osdl.org>
10 #define pr_fmt(fmt) "TCP: " fmt
12 #include <linux/module.h>
14 #include <linux/types.h>
15 #include <linux/list.h>
16 #include <linux/gfp.h>
17 #include <linux/jhash.h>
20 static DEFINE_SPINLOCK(tcp_cong_list_lock
);
21 static LIST_HEAD(tcp_cong_list
);
23 /* Simple linear search, don't expect many entries! */
24 struct tcp_congestion_ops
*tcp_ca_find(const char *name
)
26 struct tcp_congestion_ops
*e
;
28 list_for_each_entry_rcu(e
, &tcp_cong_list
, list
) {
29 if (strcmp(e
->name
, name
) == 0)
36 /* Must be called with rcu lock held */
37 static struct tcp_congestion_ops
*tcp_ca_find_autoload(struct net
*net
,
40 struct tcp_congestion_ops
*ca
= tcp_ca_find(name
);
43 if (!ca
&& capable(CAP_NET_ADMIN
)) {
45 request_module("tcp_%s", name
);
47 ca
= tcp_ca_find(name
);
53 /* Simple linear search, not much in here. */
54 struct tcp_congestion_ops
*tcp_ca_find_key(u32 key
)
56 struct tcp_congestion_ops
*e
;
58 list_for_each_entry_rcu(e
, &tcp_cong_list
, list
) {
67 * Attach new congestion control algorithm to the list
68 * of available options.
70 int tcp_register_congestion_control(struct tcp_congestion_ops
*ca
)
74 /* all algorithms must implement these */
75 if (!ca
->ssthresh
|| !ca
->undo_cwnd
||
76 !(ca
->cong_avoid
|| ca
->cong_control
)) {
77 pr_err("%s does not implement required ops\n", ca
->name
);
81 ca
->key
= jhash(ca
->name
, sizeof(ca
->name
), strlen(ca
->name
));
83 spin_lock(&tcp_cong_list_lock
);
84 if (ca
->key
== TCP_CA_UNSPEC
|| tcp_ca_find_key(ca
->key
)) {
85 pr_notice("%s already registered or non-unique key\n",
89 list_add_tail_rcu(&ca
->list
, &tcp_cong_list
);
90 pr_debug("%s registered\n", ca
->name
);
92 spin_unlock(&tcp_cong_list_lock
);
96 EXPORT_SYMBOL_GPL(tcp_register_congestion_control
);
99 * Remove congestion control algorithm, called from
100 * the module's remove function. Module ref counts are used
101 * to ensure that this can't be done till all sockets using
102 * that method are closed.
104 void tcp_unregister_congestion_control(struct tcp_congestion_ops
*ca
)
106 spin_lock(&tcp_cong_list_lock
);
107 list_del_rcu(&ca
->list
);
108 spin_unlock(&tcp_cong_list_lock
);
110 /* Wait for outstanding readers to complete before the
111 * module gets removed entirely.
113 * A try_module_get() should fail by now as our module is
114 * in "going" state since no refs are held anymore and
115 * module_exit() handler being called.
119 EXPORT_SYMBOL_GPL(tcp_unregister_congestion_control
);
121 u32
tcp_ca_get_key_by_name(struct net
*net
, const char *name
, bool *ecn_ca
)
123 const struct tcp_congestion_ops
*ca
;
124 u32 key
= TCP_CA_UNSPEC
;
129 ca
= tcp_ca_find_autoload(net
, name
);
132 *ecn_ca
= ca
->flags
& TCP_CONG_NEEDS_ECN
;
138 EXPORT_SYMBOL_GPL(tcp_ca_get_key_by_name
);
140 char *tcp_ca_get_name_by_key(u32 key
, char *buffer
)
142 const struct tcp_congestion_ops
*ca
;
146 ca
= tcp_ca_find_key(key
);
148 ret
= strncpy(buffer
, ca
->name
,
154 EXPORT_SYMBOL_GPL(tcp_ca_get_name_by_key
);
156 /* Assign choice of congestion control. */
157 void tcp_assign_congestion_control(struct sock
*sk
)
159 struct net
*net
= sock_net(sk
);
160 struct inet_connection_sock
*icsk
= inet_csk(sk
);
161 const struct tcp_congestion_ops
*ca
;
164 ca
= rcu_dereference(net
->ipv4
.tcp_congestion_control
);
165 if (unlikely(!bpf_try_module_get(ca
, ca
->owner
)))
167 icsk
->icsk_ca_ops
= ca
;
170 memset(icsk
->icsk_ca_priv
, 0, sizeof(icsk
->icsk_ca_priv
));
171 if (ca
->flags
& TCP_CONG_NEEDS_ECN
)
174 INET_ECN_dontxmit(sk
);
177 void tcp_init_congestion_control(struct sock
*sk
)
179 struct inet_connection_sock
*icsk
= inet_csk(sk
);
181 tcp_sk(sk
)->prior_ssthresh
= 0;
182 if (icsk
->icsk_ca_ops
->init
)
183 icsk
->icsk_ca_ops
->init(sk
);
184 if (tcp_ca_needs_ecn(sk
))
187 INET_ECN_dontxmit(sk
);
188 icsk
->icsk_ca_initialized
= 1;
191 static void tcp_reinit_congestion_control(struct sock
*sk
,
192 const struct tcp_congestion_ops
*ca
)
194 struct inet_connection_sock
*icsk
= inet_csk(sk
);
196 tcp_cleanup_congestion_control(sk
);
197 icsk
->icsk_ca_ops
= ca
;
198 icsk
->icsk_ca_setsockopt
= 1;
199 memset(icsk
->icsk_ca_priv
, 0, sizeof(icsk
->icsk_ca_priv
));
201 if (ca
->flags
& TCP_CONG_NEEDS_ECN
)
204 INET_ECN_dontxmit(sk
);
206 if (!((1 << sk
->sk_state
) & (TCPF_CLOSE
| TCPF_LISTEN
)))
207 tcp_init_congestion_control(sk
);
210 /* Manage refcounts on socket close. */
211 void tcp_cleanup_congestion_control(struct sock
*sk
)
213 struct inet_connection_sock
*icsk
= inet_csk(sk
);
215 if (icsk
->icsk_ca_ops
->release
)
216 icsk
->icsk_ca_ops
->release(sk
);
217 bpf_module_put(icsk
->icsk_ca_ops
, icsk
->icsk_ca_ops
->owner
);
220 /* Used by sysctl to change default congestion control */
221 int tcp_set_default_congestion_control(struct net
*net
, const char *name
)
223 struct tcp_congestion_ops
*ca
;
224 const struct tcp_congestion_ops
*prev
;
228 ca
= tcp_ca_find_autoload(net
, name
);
231 } else if (!bpf_try_module_get(ca
, ca
->owner
)) {
234 prev
= xchg(&net
->ipv4
.tcp_congestion_control
, ca
);
236 bpf_module_put(prev
, prev
->owner
);
238 ca
->flags
|= TCP_CONG_NON_RESTRICTED
;
246 /* Set default value from kernel configuration at bootup */
247 static int __init
tcp_congestion_default(void)
249 return tcp_set_default_congestion_control(&init_net
,
250 CONFIG_DEFAULT_TCP_CONG
);
252 late_initcall(tcp_congestion_default
);
254 /* Build string with list of available congestion control values */
255 void tcp_get_available_congestion_control(char *buf
, size_t maxlen
)
257 struct tcp_congestion_ops
*ca
;
261 list_for_each_entry_rcu(ca
, &tcp_cong_list
, list
) {
262 offs
+= snprintf(buf
+ offs
, maxlen
- offs
,
264 offs
== 0 ? "" : " ", ca
->name
);
266 if (WARN_ON_ONCE(offs
>= maxlen
))
272 /* Get current default congestion control */
273 void tcp_get_default_congestion_control(struct net
*net
, char *name
)
275 const struct tcp_congestion_ops
*ca
;
278 ca
= rcu_dereference(net
->ipv4
.tcp_congestion_control
);
279 strncpy(name
, ca
->name
, TCP_CA_NAME_MAX
);
283 /* Built list of non-restricted congestion control values */
284 void tcp_get_allowed_congestion_control(char *buf
, size_t maxlen
)
286 struct tcp_congestion_ops
*ca
;
291 list_for_each_entry_rcu(ca
, &tcp_cong_list
, list
) {
292 if (!(ca
->flags
& TCP_CONG_NON_RESTRICTED
))
294 offs
+= snprintf(buf
+ offs
, maxlen
- offs
,
296 offs
== 0 ? "" : " ", ca
->name
);
298 if (WARN_ON_ONCE(offs
>= maxlen
))
304 /* Change list of non-restricted congestion control */
305 int tcp_set_allowed_congestion_control(char *val
)
307 struct tcp_congestion_ops
*ca
;
308 char *saved_clone
, *clone
, *name
;
311 saved_clone
= clone
= kstrdup(val
, GFP_USER
);
315 spin_lock(&tcp_cong_list_lock
);
316 /* pass 1 check for bad entries */
317 while ((name
= strsep(&clone
, " ")) && *name
) {
318 ca
= tcp_ca_find(name
);
325 /* pass 2 clear old values */
326 list_for_each_entry_rcu(ca
, &tcp_cong_list
, list
)
327 ca
->flags
&= ~TCP_CONG_NON_RESTRICTED
;
329 /* pass 3 mark as allowed */
330 while ((name
= strsep(&val
, " ")) && *name
) {
331 ca
= tcp_ca_find(name
);
334 ca
->flags
|= TCP_CONG_NON_RESTRICTED
;
337 spin_unlock(&tcp_cong_list_lock
);
343 /* Change congestion control for socket. If load is false, then it is the
344 * responsibility of the caller to call tcp_init_congestion_control or
345 * tcp_reinit_congestion_control (if the current congestion control was
346 * already initialized.
348 int tcp_set_congestion_control(struct sock
*sk
, const char *name
, bool load
,
351 struct inet_connection_sock
*icsk
= inet_csk(sk
);
352 const struct tcp_congestion_ops
*ca
;
355 if (icsk
->icsk_ca_dst_locked
)
360 ca
= tcp_ca_find(name
);
362 ca
= tcp_ca_find_autoload(sock_net(sk
), name
);
364 /* No change asking for existing value */
365 if (ca
== icsk
->icsk_ca_ops
) {
366 icsk
->icsk_ca_setsockopt
= 1;
372 else if (!((ca
->flags
& TCP_CONG_NON_RESTRICTED
) || cap_net_admin
))
374 else if (!bpf_try_module_get(ca
, ca
->owner
))
377 tcp_reinit_congestion_control(sk
, ca
);
383 /* Slow start is used when congestion window is no greater than the slow start
384 * threshold. We base on RFC2581 and also handle stretch ACKs properly.
385 * We do not implement RFC3465 Appropriate Byte Counting (ABC) per se but
386 * something better;) a packet is only considered (s)acked in its entirety to
387 * defend the ACK attacks described in the RFC. Slow start processes a stretch
388 * ACK of degree N as if N acks of degree 1 are received back to back except
389 * ABC caps N to 2. Slow start exits when cwnd grows over ssthresh and
390 * returns the leftover acks to adjust cwnd in congestion avoidance mode.
392 u32
tcp_slow_start(struct tcp_sock
*tp
, u32 acked
)
394 u32 cwnd
= min(tp
->snd_cwnd
+ acked
, tp
->snd_ssthresh
);
396 acked
-= cwnd
- tp
->snd_cwnd
;
397 tp
->snd_cwnd
= min(cwnd
, tp
->snd_cwnd_clamp
);
401 EXPORT_SYMBOL_GPL(tcp_slow_start
);
403 /* In theory this is tp->snd_cwnd += 1 / tp->snd_cwnd (or alternative w),
404 * for every packet that was ACKed.
406 void tcp_cong_avoid_ai(struct tcp_sock
*tp
, u32 w
, u32 acked
)
408 /* If credits accumulated at a higher w, apply them gently now. */
409 if (tp
->snd_cwnd_cnt
>= w
) {
410 tp
->snd_cwnd_cnt
= 0;
414 tp
->snd_cwnd_cnt
+= acked
;
415 if (tp
->snd_cwnd_cnt
>= w
) {
416 u32 delta
= tp
->snd_cwnd_cnt
/ w
;
418 tp
->snd_cwnd_cnt
-= delta
* w
;
419 tp
->snd_cwnd
+= delta
;
421 tp
->snd_cwnd
= min(tp
->snd_cwnd
, tp
->snd_cwnd_clamp
);
423 EXPORT_SYMBOL_GPL(tcp_cong_avoid_ai
);
426 * TCP Reno congestion control
427 * This is special case used for fallback as well.
429 /* This is Jacobson's slow start and congestion avoidance.
430 * SIGCOMM '88, p. 328.
432 void tcp_reno_cong_avoid(struct sock
*sk
, u32 ack
, u32 acked
)
434 struct tcp_sock
*tp
= tcp_sk(sk
);
436 if (!tcp_is_cwnd_limited(sk
))
439 /* In "safe" area, increase. */
440 if (tcp_in_slow_start(tp
)) {
441 acked
= tcp_slow_start(tp
, acked
);
445 /* In dangerous area, increase slowly. */
446 tcp_cong_avoid_ai(tp
, tp
->snd_cwnd
, acked
);
448 EXPORT_SYMBOL_GPL(tcp_reno_cong_avoid
);
450 /* Slow start threshold is half the congestion window (min 2) */
451 u32
tcp_reno_ssthresh(struct sock
*sk
)
453 const struct tcp_sock
*tp
= tcp_sk(sk
);
455 return max(tp
->snd_cwnd
>> 1U, 2U);
457 EXPORT_SYMBOL_GPL(tcp_reno_ssthresh
);
459 u32
tcp_reno_undo_cwnd(struct sock
*sk
)
461 const struct tcp_sock
*tp
= tcp_sk(sk
);
463 return max(tp
->snd_cwnd
, tp
->prior_cwnd
);
465 EXPORT_SYMBOL_GPL(tcp_reno_undo_cwnd
);
467 struct tcp_congestion_ops tcp_reno
= {
468 .flags
= TCP_CONG_NON_RESTRICTED
,
470 .owner
= THIS_MODULE
,
471 .ssthresh
= tcp_reno_ssthresh
,
472 .cong_avoid
= tcp_reno_cong_avoid
,
473 .undo_cwnd
= tcp_reno_undo_cwnd
,