]>
Commit | Line | Data |
---|---|---|
10467163 | 1 | #include <linux/err.h> |
2100c8d2 YC |
2 | #include <linux/init.h> |
3 | #include <linux/kernel.h> | |
10467163 JC |
4 | #include <linux/list.h> |
5 | #include <linux/tcp.h> | |
6 | #include <linux/rcupdate.h> | |
7 | #include <linux/rculist.h> | |
8 | #include <net/inetpeer.h> | |
9 | #include <net/tcp.h> | |
2100c8d2 | 10 | |
0d41cca4 | 11 | int sysctl_tcp_fastopen __read_mostly = TFO_CLIENT_ENABLE; |
10467163 JC |
12 | |
13 | struct tcp_fastopen_context __rcu *tcp_fastopen_ctx; | |
14 | ||
15 | static DEFINE_SPINLOCK(tcp_fastopen_ctx_lock); | |
16 | ||
222e83d2 HFS |
17 | void tcp_fastopen_init_key_once(bool publish) |
18 | { | |
19 | static u8 key[TCP_FASTOPEN_KEY_LENGTH]; | |
20 | ||
21 | /* tcp_fastopen_reset_cipher publishes the new context | |
22 | * atomically, so we allow this race happening here. | |
23 | * | |
24 | * All call sites of tcp_fastopen_cookie_gen also check | |
25 | * for a valid cookie, so this is an acceptable risk. | |
26 | */ | |
27 | if (net_get_random_once(key, sizeof(key)) && publish) | |
28 | tcp_fastopen_reset_cipher(key, sizeof(key)); | |
29 | } | |
30 | ||
10467163 JC |
31 | static void tcp_fastopen_ctx_free(struct rcu_head *head) |
32 | { | |
33 | struct tcp_fastopen_context *ctx = | |
34 | container_of(head, struct tcp_fastopen_context, rcu); | |
35 | crypto_free_cipher(ctx->tfm); | |
36 | kfree(ctx); | |
37 | } | |
38 | ||
39 | int tcp_fastopen_reset_cipher(void *key, unsigned int len) | |
40 | { | |
41 | int err; | |
42 | struct tcp_fastopen_context *ctx, *octx; | |
43 | ||
44 | ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); | |
45 | if (!ctx) | |
46 | return -ENOMEM; | |
47 | ctx->tfm = crypto_alloc_cipher("aes", 0, 0); | |
48 | ||
49 | if (IS_ERR(ctx->tfm)) { | |
50 | err = PTR_ERR(ctx->tfm); | |
51 | error: kfree(ctx); | |
52 | pr_err("TCP: TFO aes cipher alloc error: %d\n", err); | |
53 | return err; | |
54 | } | |
55 | err = crypto_cipher_setkey(ctx->tfm, key, len); | |
56 | if (err) { | |
57 | pr_err("TCP: TFO cipher key error: %d\n", err); | |
58 | crypto_free_cipher(ctx->tfm); | |
59 | goto error; | |
60 | } | |
61 | memcpy(ctx->key, key, len); | |
62 | ||
63 | spin_lock(&tcp_fastopen_ctx_lock); | |
64 | ||
65 | octx = rcu_dereference_protected(tcp_fastopen_ctx, | |
66 | lockdep_is_held(&tcp_fastopen_ctx_lock)); | |
67 | rcu_assign_pointer(tcp_fastopen_ctx, ctx); | |
68 | spin_unlock(&tcp_fastopen_ctx_lock); | |
69 | ||
70 | if (octx) | |
71 | call_rcu(&octx->rcu, tcp_fastopen_ctx_free); | |
72 | return err; | |
73 | } | |
74 | ||
3a19ce0e DL |
75 | static bool __tcp_fastopen_cookie_gen(const void *path, |
76 | struct tcp_fastopen_cookie *foc) | |
10467163 | 77 | { |
10467163 | 78 | struct tcp_fastopen_context *ctx; |
3a19ce0e | 79 | bool ok = false; |
10467163 | 80 | |
222e83d2 HFS |
81 | tcp_fastopen_init_key_once(true); |
82 | ||
10467163 JC |
83 | rcu_read_lock(); |
84 | ctx = rcu_dereference(tcp_fastopen_ctx); | |
85 | if (ctx) { | |
3a19ce0e | 86 | crypto_cipher_encrypt_one(ctx->tfm, foc->val, path); |
10467163 | 87 | foc->len = TCP_FASTOPEN_COOKIE_SIZE; |
3a19ce0e | 88 | ok = true; |
10467163 JC |
89 | } |
90 | rcu_read_unlock(); | |
3a19ce0e DL |
91 | return ok; |
92 | } | |
93 | ||
94 | /* Generate the fastopen cookie by doing aes128 encryption on both | |
95 | * the source and destination addresses. Pad 0s for IPv4 or IPv4-mapped-IPv6 | |
96 | * addresses. For the longer IPv6 addresses use CBC-MAC. | |
97 | * | |
98 | * XXX (TFO) - refactor when TCP_FASTOPEN_COOKIE_SIZE != AES_BLOCK_SIZE. | |
99 | */ | |
100 | static bool tcp_fastopen_cookie_gen(struct request_sock *req, | |
101 | struct sk_buff *syn, | |
102 | struct tcp_fastopen_cookie *foc) | |
103 | { | |
104 | if (req->rsk_ops->family == AF_INET) { | |
105 | const struct iphdr *iph = ip_hdr(syn); | |
106 | ||
107 | __be32 path[4] = { iph->saddr, iph->daddr, 0, 0 }; | |
108 | return __tcp_fastopen_cookie_gen(path, foc); | |
109 | } | |
110 | ||
111 | #if IS_ENABLED(CONFIG_IPV6) | |
112 | if (req->rsk_ops->family == AF_INET6) { | |
113 | const struct ipv6hdr *ip6h = ipv6_hdr(syn); | |
114 | struct tcp_fastopen_cookie tmp; | |
115 | ||
116 | if (__tcp_fastopen_cookie_gen(&ip6h->saddr, &tmp)) { | |
117 | struct in6_addr *buf = (struct in6_addr *) tmp.val; | |
41c91996 | 118 | int i; |
3a19ce0e DL |
119 | |
120 | for (i = 0; i < 4; i++) | |
121 | buf->s6_addr32[i] ^= ip6h->daddr.s6_addr32[i]; | |
122 | return __tcp_fastopen_cookie_gen(buf, foc); | |
123 | } | |
124 | } | |
125 | #endif | |
126 | return false; | |
10467163 | 127 | } |
5b7ed089 | 128 | |
843f4a55 YC |
129 | static bool tcp_fastopen_create_child(struct sock *sk, |
130 | struct sk_buff *skb, | |
131 | struct dst_entry *dst, | |
132 | struct request_sock *req) | |
5b7ed089 | 133 | { |
17846376 | 134 | struct tcp_sock *tp; |
5b7ed089 | 135 | struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue; |
5b7ed089 | 136 | struct sock *child; |
ba34e6d9 | 137 | u32 end_seq; |
5b7ed089 YC |
138 | |
139 | req->num_retrans = 0; | |
140 | req->num_timeout = 0; | |
141 | req->sk = NULL; | |
142 | ||
143 | child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL); | |
51456b29 | 144 | if (!child) |
843f4a55 | 145 | return false; |
5b7ed089 YC |
146 | |
147 | spin_lock(&queue->fastopenq->lock); | |
148 | queue->fastopenq->qlen++; | |
149 | spin_unlock(&queue->fastopenq->lock); | |
150 | ||
151 | /* Initialize the child socket. Have to fix some values to take | |
152 | * into account the child is a Fast Open socket and is created | |
153 | * only out of the bits carried in the SYN packet. | |
154 | */ | |
155 | tp = tcp_sk(child); | |
156 | ||
157 | tp->fastopen_rsk = req; | |
9439ce00 | 158 | tcp_rsk(req)->tfo_listener = true; |
5b7ed089 YC |
159 | |
160 | /* RFC1323: The window in SYN & SYN/ACK segments is never | |
161 | * scaled. So correct it appropriately. | |
162 | */ | |
163 | tp->snd_wnd = ntohs(tcp_hdr(skb)->window); | |
164 | ||
165 | /* Activate the retrans timer so that SYNACK can be retransmitted. | |
166 | * The request socket is not added to the SYN table of the parent | |
167 | * because it's been added to the accept queue directly. | |
168 | */ | |
169 | inet_csk_reset_xmit_timer(child, ICSK_TIME_RETRANS, | |
170 | TCP_TIMEOUT_INIT, TCP_RTO_MAX); | |
171 | ||
0470c8ca | 172 | atomic_set(&req->rsk_refcnt, 1); |
5b7ed089 YC |
173 | /* Add the child socket directly into the accept queue */ |
174 | inet_csk_reqsk_queue_add(sk, req, child); | |
175 | ||
176 | /* Now finish processing the fastopen child socket. */ | |
177 | inet_csk(child)->icsk_af_ops->rebuild_header(child); | |
178 | tcp_init_congestion_control(child); | |
179 | tcp_mtup_init(child); | |
180 | tcp_init_metrics(child); | |
181 | tcp_init_buffer_space(child); | |
182 | ||
183 | /* Queue the data carried in the SYN packet. We need to first | |
184 | * bump skb's refcnt because the caller will attempt to free it. | |
ba34e6d9 ED |
185 | * Note that IPv6 might also have used skb_get() trick |
186 | * in tcp_v6_conn_request() to keep this SYN around (treq->pktopts) | |
187 | * So we need to eventually get a clone of the packet, | |
188 | * before inserting it in sk_receive_queue. | |
5b7ed089 | 189 | * |
843f4a55 YC |
190 | * XXX (TFO) - we honor a zero-payload TFO request for now, |
191 | * (any reason not to?) but no need to queue the skb since | |
192 | * there is no data. How about SYN+FIN? | |
5b7ed089 | 193 | */ |
ba34e6d9 ED |
194 | end_seq = TCP_SKB_CB(skb)->end_seq; |
195 | if (end_seq != TCP_SKB_CB(skb)->seq + 1) { | |
196 | struct sk_buff *skb2; | |
197 | ||
198 | if (unlikely(skb_shared(skb))) | |
199 | skb2 = skb_clone(skb, GFP_ATOMIC); | |
200 | else | |
201 | skb2 = skb_get(skb); | |
202 | ||
203 | if (likely(skb2)) { | |
204 | skb_dst_drop(skb2); | |
205 | __skb_pull(skb2, tcp_hdrlen(skb)); | |
206 | skb_set_owner_r(skb2, child); | |
207 | __skb_queue_tail(&child->sk_receive_queue, skb2); | |
208 | tp->syn_data_acked = 1; | |
d654976c ED |
209 | |
210 | /* u64_stats_update_begin(&tp->syncp) not needed here, | |
211 | * as we certainly are not changing upper 32bit value (0) | |
212 | */ | |
bdd1f9ed | 213 | tp->bytes_received = end_seq - TCP_SKB_CB(skb)->seq - 1; |
ba34e6d9 ED |
214 | } else { |
215 | end_seq = TCP_SKB_CB(skb)->seq + 1; | |
216 | } | |
5b7ed089 | 217 | } |
ba34e6d9 | 218 | tcp_rsk(req)->rcv_nxt = tp->rcv_nxt = end_seq; |
5b7ed089 YC |
219 | sk->sk_data_ready(sk); |
220 | bh_unlock_sock(child); | |
221 | sock_put(child); | |
51456b29 | 222 | WARN_ON(!req->sk); |
843f4a55 | 223 | return true; |
5b7ed089 | 224 | } |
5b7ed089 YC |
225 | |
226 | static bool tcp_fastopen_queue_check(struct sock *sk) | |
227 | { | |
228 | struct fastopen_queue *fastopenq; | |
229 | ||
230 | /* Make sure the listener has enabled fastopen, and we don't | |
231 | * exceed the max # of pending TFO requests allowed before trying | |
232 | * to validating the cookie in order to avoid burning CPU cycles | |
233 | * unnecessarily. | |
234 | * | |
235 | * XXX (TFO) - The implication of checking the max_qlen before | |
236 | * processing a cookie request is that clients can't differentiate | |
237 | * between qlen overflow causing Fast Open to be disabled | |
238 | * temporarily vs a server not supporting Fast Open at all. | |
239 | */ | |
240 | fastopenq = inet_csk(sk)->icsk_accept_queue.fastopenq; | |
51456b29 | 241 | if (!fastopenq || fastopenq->max_qlen == 0) |
5b7ed089 YC |
242 | return false; |
243 | ||
244 | if (fastopenq->qlen >= fastopenq->max_qlen) { | |
245 | struct request_sock *req1; | |
246 | spin_lock(&fastopenq->lock); | |
247 | req1 = fastopenq->rskq_rst_head; | |
fa76ce73 | 248 | if (!req1 || time_after(req1->rsk_timer.expires, jiffies)) { |
5b7ed089 YC |
249 | spin_unlock(&fastopenq->lock); |
250 | NET_INC_STATS_BH(sock_net(sk), | |
251 | LINUX_MIB_TCPFASTOPENLISTENOVERFLOW); | |
252 | return false; | |
253 | } | |
254 | fastopenq->rskq_rst_head = req1->dl_next; | |
255 | fastopenq->qlen--; | |
256 | spin_unlock(&fastopenq->lock); | |
13854e5a | 257 | reqsk_put(req1); |
5b7ed089 YC |
258 | } |
259 | return true; | |
260 | } | |
261 | ||
89278c9d YC |
262 | /* Returns true if we should perform Fast Open on the SYN. The cookie (foc) |
263 | * may be updated and return the client in the SYN-ACK later. E.g., Fast Open | |
264 | * cookie request (foc->len == 0). | |
265 | */ | |
843f4a55 YC |
266 | bool tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, |
267 | struct request_sock *req, | |
268 | struct tcp_fastopen_cookie *foc, | |
269 | struct dst_entry *dst) | |
5b7ed089 | 270 | { |
89278c9d YC |
271 | struct tcp_fastopen_cookie valid_foc = { .len = -1 }; |
272 | bool syn_data = TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq + 1; | |
5b7ed089 | 273 | |
531c94a9 YC |
274 | if (foc->len == 0) /* Client requests a cookie */ |
275 | NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENCOOKIEREQD); | |
276 | ||
89278c9d YC |
277 | if (!((sysctl_tcp_fastopen & TFO_SERVER_ENABLE) && |
278 | (syn_data || foc->len >= 0) && | |
279 | tcp_fastopen_queue_check(sk))) { | |
280 | foc->len = -1; | |
5b7ed089 | 281 | return false; |
5b7ed089 YC |
282 | } |
283 | ||
89278c9d YC |
284 | if (syn_data && (sysctl_tcp_fastopen & TFO_SERVER_COOKIE_NOT_REQD)) |
285 | goto fastopen; | |
286 | ||
531c94a9 YC |
287 | if (foc->len >= 0 && /* Client presents or requests a cookie */ |
288 | tcp_fastopen_cookie_gen(req, skb, &valid_foc) && | |
3a19ce0e | 289 | foc->len == TCP_FASTOPEN_COOKIE_SIZE && |
89278c9d YC |
290 | foc->len == valid_foc.len && |
291 | !memcmp(foc->val, valid_foc.val, foc->len)) { | |
843f4a55 YC |
292 | /* Cookie is valid. Create a (full) child socket to accept |
293 | * the data in SYN before returning a SYN-ACK to ack the | |
294 | * data. If we fail to create the socket, fall back and | |
295 | * ack the ISN only but includes the same cookie. | |
296 | * | |
297 | * Note: Data-less SYN with valid cookie is allowed to send | |
298 | * data in SYN_RECV state. | |
299 | */ | |
89278c9d | 300 | fastopen: |
843f4a55 YC |
301 | if (tcp_fastopen_create_child(sk, skb, dst, req)) { |
302 | foc->len = -1; | |
303 | NET_INC_STATS_BH(sock_net(sk), | |
304 | LINUX_MIB_TCPFASTOPENPASSIVE); | |
305 | return true; | |
306 | } | |
531c94a9 YC |
307 | NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENPASSIVEFAIL); |
308 | } else if (foc->len > 0) /* Client presents an invalid cookie */ | |
309 | NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENPASSIVEFAIL); | |
89278c9d | 310 | |
7f9b838b | 311 | valid_foc.exp = foc->exp; |
89278c9d | 312 | *foc = valid_foc; |
5b7ed089 YC |
313 | return false; |
314 | } | |
843f4a55 | 315 | EXPORT_SYMBOL(tcp_try_fastopen); |