]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/crypto/chelsio/chtls/chtls_cm.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto/chtls: IPv6 support for inline TLS
[mirror_ubuntu-jammy-kernel.git] / drivers / crypto / chelsio / chtls / chtls_cm.c
CommitLineData
d2912cb1 1// SPDX-License-Identifier: GPL-2.0-only
cc35c88a
AG		 2/*
3 * Copyright (c) 2018 Chelsio Communications, Inc.
4 *
cc35c88a
AG		 5 * Written by: Atul Gupta (atul.gupta@chelsio.com)
6 */
7
8#include <linux/module.h>
9#include <linux/list.h>
10#include <linux/workqueue.h>
11#include <linux/skbuff.h>
12#include <linux/timer.h>
13#include <linux/notifier.h>
14#include <linux/inetdevice.h>
15#include <linux/ip.h>
16#include <linux/tcp.h>
17#include <linux/sched/signal.h>
18#include <linux/kallsyms.h>
19#include <linux/kprobes.h>
20#include <linux/if_vlan.h>
6abde0b2
VKY		 21#include <linux/ipv6.h>
22#include <net/ipv6.h>
23#include <net/transp_v6.h>
24#include <net/ip6_route.h>
0c3a16be 25#include <net/inet_common.h>
cc35c88a
AG		 26#include <net/tcp.h>
27#include <net/dst.h>
76f7164d 28#include <net/tls.h>
6abde0b2
VKY		 29#include <net/addrconf.h>
30#include <net/secure_seq.h>
cc35c88a
AG		 31
32#include "chtls.h"
33#include "chtls_cm.h"
6abde0b2 34#include "clip_tbl.h"
cc35c88a
AG		 35
36/*
37 * State transitions and actions for close. Note that if we are in SYN_SENT
38 * we remain in that state as we cannot control a connection while it's in
39 * SYN_SENT; such connections are allowed to establish and are then aborted.
40 */
41static unsigned char new_state[16] = {
42 /* current state: new state: action: */
43 /* (Invalid) */ TCP_CLOSE,
44 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
45 /* TCP_SYN_SENT */ TCP_SYN_SENT,
46 /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
47 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
48 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
49 /* TCP_TIME_WAIT */ TCP_CLOSE,
50 /* TCP_CLOSE */ TCP_CLOSE,
51 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
52 /* TCP_LAST_ACK */ TCP_LAST_ACK,
53 /* TCP_LISTEN */ TCP_CLOSE,
54 /* TCP_CLOSING */ TCP_CLOSING,
55};
56
57static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
58{
59 struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
60
61 if (!csk)
62 return NULL;
63
64 csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
65 if (!csk->txdata_skb_cache) {
66 kfree(csk);
67 return NULL;
68 }
69
70 kref_init(&csk->kref);
71 csk->cdev = cdev;
72 skb_queue_head_init(&csk->txq);
73 csk->wr_skb_head = NULL;
74 csk->wr_skb_tail = NULL;
75 csk->mss = MAX_MSS;
76 csk->tlshws.ofld = 1;
77 csk->tlshws.txkey = -1;
78 csk->tlshws.rxkey = -1;
79 csk->tlshws.mfs = TLS_MFS;
80 skb_queue_head_init(&csk->tlshws.sk_recv_queue);
81 return csk;
82}
83
84static void chtls_sock_release(struct kref *ref)
85{
86 struct chtls_sock *csk =
87 container_of(ref, struct chtls_sock, kref);
88
89 kfree(csk);
90}
91
6abde0b2 92static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,
cc35c88a
AG		 93 struct sock *sk)
94{
95 struct net_device *ndev = cdev->ports[0];
6abde0b2
VKY		 96 struct net_device *temp;
97 int addr_type;
98
99 switch (sk->sk_family) {
100 case PF_INET:
101 if (likely(!inet_sk(sk)->inet_rcv_saddr))
102 return ndev;
103 ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
104 break;
105 case PF_INET6:
106 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
107 if (likely(addr_type == IPV6_ADDR_ANY))
108 return ndev;
109
110 for_each_netdev_rcu(&init_net, temp) {
111 if (ipv6_chk_addr(&init_net, (struct in6_addr *)
112 &sk->sk_v6_rcv_saddr, temp, 1)) {
113 ndev = temp;
114 break;
115 }
116 }
117 break;
118 default:
119 return NULL;
120 }
cc35c88a 121
cc35c88a
AG		 122 if (!ndev)
123 return NULL;
124
125 if (is_vlan_dev(ndev))
126 return vlan_dev_real_dev(ndev);
127 return ndev;
128}
129
130static void assign_rxopt(struct sock *sk, unsigned int opt)
131{
132 const struct chtls_dev *cdev;
133 struct chtls_sock *csk;
134 struct tcp_sock *tp;
135
136 csk = rcu_dereference_sk_user_data(sk);
137 tp = tcp_sk(sk);
138
139 cdev = csk->cdev;
140 tp->tcp_header_len = sizeof(struct tcphdr);
141 tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
142 tp->mss_cache = tp->rx_opt.mss_clamp;
143 tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
144 tp->rx_opt.snd_wscale = TCPOPT_SACK_G(opt);
145 tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
146 SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
147 if (!tp->rx_opt.wscale_ok)
148 tp->rx_opt.rcv_wscale = 0;
149 if (tp->rx_opt.tstamp_ok) {
150 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
151 tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
152 } else if (csk->opt2 & TSTAMPS_EN_F) {
153 csk->opt2 &= ~TSTAMPS_EN_F;
154 csk->mtu_idx = TCPOPT_MSS_G(opt);
155 }
156}
157
158static void chtls_purge_receive_queue(struct sock *sk)
159{
160 struct sk_buff *skb;
161
162 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
163 skb_dst_set(skb, (void *)NULL);
164 kfree_skb(skb);
165 }
166}
167
168static void chtls_purge_write_queue(struct sock *sk)
169{
170 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
171 struct sk_buff *skb;
172
173 while ((skb = __skb_dequeue(&csk->txq))) {
174 sk->sk_wmem_queued -= skb->truesize;
175 __kfree_skb(skb);
176 }
177}
178
179static void chtls_purge_recv_queue(struct sock *sk)
180{
181 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
182 struct chtls_hws *tlsk = &csk->tlshws;
183 struct sk_buff *skb;
184
185 while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
186 skb_dst_set(skb, NULL);
187 kfree_skb(skb);
188 }
189}
190
191static void abort_arp_failure(void *handle, struct sk_buff *skb)
192{
193 struct cpl_abort_req *req = cplhdr(skb);
194 struct chtls_dev *cdev;
195
196 cdev = (struct chtls_dev *)handle;
197 req->cmd = CPL_ABORT_NO_RST;
198 cxgb4_ofld_send(cdev->lldi->ports[0], skb);
199}
200
201static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
202{
203 if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
204 __skb_trim(skb, 0);
205 refcount_add(2, &skb->users);
206 } else {
207 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
208 }
209 return skb;
210}
211
212static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
213{
214 struct cpl_abort_req *req;
215 struct chtls_sock *csk;
216 struct tcp_sock *tp;
217
218 csk = rcu_dereference_sk_user_data(sk);
219 tp = tcp_sk(sk);
220
221 if (!skb)
222 skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
223
224 req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
225 INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
226 skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
227 req->rsvd0 = htonl(tp->snd_nxt);
228 req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
229 req->cmd = mode;
230 t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
231 send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
232}
233
234static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
235{
236 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
237
238 if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
239 !csk->cdev)) {
240 if (sk->sk_state == TCP_SYN_RECV)
241 csk_set_flag(csk, CSK_RST_ABORTED);
242 goto out;
243 }
244
245 if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
246 struct tcp_sock *tp = tcp_sk(sk);
247
248 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
249 WARN_ONCE(1, "send tx flowc error");
250 csk_set_flag(csk, CSK_TX_DATA_SENT);
251 }
252
253 csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
254 chtls_purge_write_queue(sk);
255
256 csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
257 if (sk->sk_state != TCP_SYN_RECV)
258 chtls_send_abort(sk, mode, skb);
259 else
260 goto out;
261
262 return;
263out:
ce1294d9 264 kfree_skb(skb);
cc35c88a
AG		 265}
266
267static void release_tcp_port(struct sock *sk)
268{
269 if (inet_csk(sk)->icsk_bind_hash)
270 inet_put_port(sk);
271}
272
273static void tcp_uncork(struct sock *sk)
274{
275 struct tcp_sock *tp = tcp_sk(sk);
276
277 if (tp->nonagle & TCP_NAGLE_CORK) {
278 tp->nonagle &= ~TCP_NAGLE_CORK;
279 chtls_tcp_push(sk, 0);
280 }
281}
282
283static void chtls_close_conn(struct sock *sk)
284{
285 struct cpl_close_con_req *req;
286 struct chtls_sock *csk;
287 struct sk_buff *skb;
288 unsigned int tid;
289 unsigned int len;
290
291 len = roundup(sizeof(struct cpl_close_con_req), 16);
292 csk = rcu_dereference_sk_user_data(sk);
293 tid = csk->tid;
294
295 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
296 req = (struct cpl_close_con_req *)__skb_put(skb, len);
297 memset(req, 0, len);
298 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
299 FW_WR_IMMDLEN_V(sizeof(*req) -
300 sizeof(req->wr)));
301 req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
302 FW_WR_FLOWID_V(tid));
303
304 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
305
306 tcp_uncork(sk);
307 skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
308 if (sk->sk_state != TCP_SYN_SENT)
309 chtls_push_frames(csk, 1);
310}
311
312/*
313 * Perform a state transition during close and return the actions indicated
314 * for the transition. Do not make this function inline, the main reason
315 * it exists at all is to avoid multiple inlining of tcp_set_state.
316 */
317static int make_close_transition(struct sock *sk)
318{
319 int next = (int)new_state[sk->sk_state];
320
321 tcp_set_state(sk, next & TCP_STATE_MASK);
322 return next & TCP_ACTION_FIN;
323}
324
325void chtls_close(struct sock *sk, long timeout)
326{
327 int data_lost, prev_state;
328 struct chtls_sock *csk;
329
330 csk = rcu_dereference_sk_user_data(sk);
331
332 lock_sock(sk);
333 sk->sk_shutdown |= SHUTDOWN_MASK;
334
335 data_lost = skb_queue_len(&sk->sk_receive_queue);
336 data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
337 chtls_purge_recv_queue(sk);
338 chtls_purge_receive_queue(sk);
339
340 if (sk->sk_state == TCP_CLOSE) {
341 goto wait;
342 } else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
343 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
344 release_tcp_port(sk);
345 goto unlock;
346 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
347 sk->sk_prot->disconnect(sk, 0);
348 } else if (make_close_transition(sk)) {
349 chtls_close_conn(sk);
350 }
351wait:
352 if (timeout)
353 sk_stream_wait_close(sk, timeout);
354
355unlock:
356 prev_state = sk->sk_state;
357 sock_hold(sk);
358 sock_orphan(sk);
359
360 release_sock(sk);
361
362 local_bh_disable();
363 bh_lock_sock(sk);
364
365 if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
366 goto out;
367
368 if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
369 !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
370 struct sk_buff *skb;
371
372 skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
373 if (skb)
374 chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
375 }
376
377 if (sk->sk_state == TCP_CLOSE)
378 inet_csk_destroy_sock(sk);
379
380out:
381 bh_unlock_sock(sk);
382 local_bh_enable();
383 sock_put(sk);
384}
385
386/*
387 * Wait until a socket enters on of the given states.
388 */
389static int wait_for_states(struct sock *sk, unsigned int states)
390{
391 DECLARE_WAITQUEUE(wait, current);
392 struct socket_wq _sk_wq;
393 long current_timeo;
394 int err = 0;
395
396 current_timeo = 200;
397
398 /*
399 * We want this to work even when there's no associated struct socket.
400 * In that case we provide a temporary wait_queue_head_t.
401 */
402 if (!sk->sk_wq) {
403 init_waitqueue_head(&_sk_wq.wait);
404 _sk_wq.fasync_list = NULL;
405 init_rcu_head_on_stack(&_sk_wq.rcu);
406 RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
407 }
408
409 add_wait_queue(sk_sleep(sk), &wait);
410 while (!sk_in_state(sk, states)) {
411 if (!current_timeo) {
412 err = -EBUSY;
413 break;
414 }
415 if (signal_pending(current)) {
416 err = sock_intr_errno(current_timeo);
417 break;
418 }
419 set_current_state(TASK_UNINTERRUPTIBLE);
420 release_sock(sk);
421 if (!sk_in_state(sk, states))
422 current_timeo = schedule_timeout(current_timeo);
423 __set_current_state(TASK_RUNNING);
424 lock_sock(sk);
425 }
426 remove_wait_queue(sk_sleep(sk), &wait);
427
428 if (rcu_dereference(sk->sk_wq) == &_sk_wq)
429 sk->sk_wq = NULL;
430 return err;
431}
432
433int chtls_disconnect(struct sock *sk, int flags)
434{
cc35c88a
AG		 435 struct tcp_sock *tp;
436 int err;
437
438 tp = tcp_sk(sk);
cc35c88a
AG		 439 chtls_purge_recv_queue(sk);
440 chtls_purge_receive_queue(sk);
441 chtls_purge_write_queue(sk);
442
443 if (sk->sk_state != TCP_CLOSE) {
444 sk->sk_err = ECONNRESET;
445 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
446 err = wait_for_states(sk, TCPF_CLOSE);
447 if (err)
448 return err;
449 }
450 chtls_purge_recv_queue(sk);
451 chtls_purge_receive_queue(sk);
452 tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
453 return tcp_disconnect(sk, flags);
454}
455
456#define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
457 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
458void chtls_shutdown(struct sock *sk, int how)
459{
460 if ((how & SEND_SHUTDOWN) &&
461 sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
462 make_close_transition(sk))
463 chtls_close_conn(sk);
464}
465
466void chtls_destroy_sock(struct sock *sk)
467{
468 struct chtls_sock *csk;
469
470 csk = rcu_dereference_sk_user_data(sk);
471 chtls_purge_recv_queue(sk);
472 csk->ulp_mode = ULP_MODE_NONE;
473 chtls_purge_write_queue(sk);
474 free_tls_keyid(sk);
475 kref_put(&csk->kref, chtls_sock_release);
3a0a9783 476 csk->cdev = NULL;
6abde0b2
VKY		 477 if (sk->sk_family == AF_INET)
478 sk->sk_prot = &tcp_prot;
479 else
480 sk->sk_prot = &tcpv6_prot;
cc35c88a
AG		 481 sk->sk_prot->destroy(sk);
482}
483
484static void reset_listen_child(struct sock *child)
485{
486 struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
487 struct sk_buff *skb;
488
489 skb = alloc_ctrl_skb(csk->txdata_skb_cache,
490 sizeof(struct cpl_abort_req));
491
492 chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
493 sock_orphan(child);
494 INC_ORPHAN_COUNT(child);
495 if (child->sk_state == TCP_CLOSE)
496 inet_csk_destroy_sock(child);
497}
498
499static void chtls_disconnect_acceptq(struct sock *listen_sk)
500{
501 struct request_sock **pprev;
502
503 pprev = ACCEPT_QUEUE(listen_sk);
504 while (*pprev) {
505 struct request_sock *req = *pprev;
506
6abde0b2
VKY		 507 if (req->rsk_ops == &chtls_rsk_ops ||
508 req->rsk_ops == &chtls_rsk_opsv6) {
cc35c88a
AG		 509 struct sock *child = req->sk;
510
511 *pprev = req->dl_next;
512 sk_acceptq_removed(listen_sk);
513 reqsk_put(req);
514 sock_hold(child);
515 local_bh_disable();
516 bh_lock_sock(child);
517 release_tcp_port(child);
518 reset_listen_child(child);
519 bh_unlock_sock(child);
520 local_bh_enable();
521 sock_put(child);
522 } else {
523 pprev = &req->dl_next;
524 }
525 }
526}
527
528static int listen_hashfn(const struct sock *sk)
529{
530 return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
531}
532
533static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
534 struct sock *sk,
535 unsigned int stid)
536{
537 struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL);
538
539 if (p) {
540 int key = listen_hashfn(sk);
541
542 p->sk = sk;
543 p->stid = stid;
544 spin_lock(&cdev->listen_lock);
545 p->next = cdev->listen_hash_tab[key];
546 cdev->listen_hash_tab[key] = p;
547 spin_unlock(&cdev->listen_lock);
548 }
549 return p;
550}
551
552static int listen_hash_find(struct chtls_dev *cdev,
553 struct sock *sk)
554{
555 struct listen_info *p;
556 int stid = -1;
557 int key;
558
559 key = listen_hashfn(sk);
560
561 spin_lock(&cdev->listen_lock);
562 for (p = cdev->listen_hash_tab[key]; p; p = p->next)
563 if (p->sk == sk) {
564 stid = p->stid;
565 break;
566 }
567 spin_unlock(&cdev->listen_lock);
568 return stid;
569}
570
571static int listen_hash_del(struct chtls_dev *cdev,
572 struct sock *sk)
573{
574 struct listen_info *p, **prev;
575 int stid = -1;
576 int key;
577
578 key = listen_hashfn(sk);
579 prev = &cdev->listen_hash_tab[key];
580
581 spin_lock(&cdev->listen_lock);
582 for (p = *prev; p; prev = &p->next, p = p->next)
583 if (p->sk == sk) {
584 stid = p->stid;
585 *prev = p->next;
586 kfree(p);
587 break;
588 }
589 spin_unlock(&cdev->listen_lock);
590 return stid;
591}
592
593static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
594{
595 struct request_sock *req;
596 struct chtls_sock *csk;
597
598 csk = rcu_dereference_sk_user_data(child);
599 req = csk->passive_reap_next;
600
601 reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
602 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
603 chtls_reqsk_free(req);
604 csk->passive_reap_next = NULL;
605}
606
607static void chtls_reset_synq(struct listen_ctx *listen_ctx)
608{
609 struct sock *listen_sk = listen_ctx->lsk;
610
611 while (!skb_queue_empty(&listen_ctx->synq)) {
612 struct chtls_sock *csk =
613 container_of((struct synq *)__skb_dequeue
614 (&listen_ctx->synq), struct chtls_sock, synq);
615 struct sock *child = csk->sk;
616
617 cleanup_syn_rcv_conn(child, listen_sk);
618 sock_hold(child);
619 local_bh_disable();
620 bh_lock_sock(child);
621 release_tcp_port(child);
622 reset_listen_child(child);
623 bh_unlock_sock(child);
624 local_bh_enable();
625 sock_put(child);
626 }
627}
628
629int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
630{
631 struct net_device *ndev;
632 struct listen_ctx *ctx;
633 struct adapter *adap;
634 struct port_info *pi;
6abde0b2 635 bool clip_valid;
cc35c88a
AG		 636 int stid;
637 int ret;
638
6abde0b2 639 clip_valid = false;
cc35c88a 640 rcu_read_lock();
6abde0b2 641 ndev = chtls_find_netdev(cdev, sk);
cc35c88a
AG		 642 rcu_read_unlock();
643 if (!ndev)
644 return -EBADF;
645
646 pi = netdev_priv(ndev);
647 adap = pi->adapter;
80f61f19 648 if (!(adap->flags & CXGB4_FULL_INIT_DONE))
cc35c88a
AG		 649 return -EBADF;
650
651 if (listen_hash_find(cdev, sk) >= 0) /* already have it */
652 return -EADDRINUSE;
653
654 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
655 if (!ctx)
656 return -ENOMEM;
657
658 __module_get(THIS_MODULE);
659 ctx->lsk = sk;
660 ctx->cdev = cdev;
661 ctx->state = T4_LISTEN_START_PENDING;
662 skb_queue_head_init(&ctx->synq);
663
664 stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
665 if (stid < 0)
666 goto free_ctx;
667
668 sock_hold(sk);
669 if (!listen_hash_add(cdev, sk, stid))
670 goto free_stid;
671
6abde0b2
VKY		 672 if (sk->sk_family == PF_INET) {
673 ret = cxgb4_create_server(ndev, stid,
674 inet_sk(sk)->inet_rcv_saddr,
675 inet_sk(sk)->inet_sport, 0,
676 cdev->lldi->rxq_ids[0]);
677 } else {
678 int addr_type;
679
680 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
681 if (addr_type != IPV6_ADDR_ANY) {
682 ret = cxgb4_clip_get(ndev, (const u32 *)
683 &sk->sk_v6_rcv_saddr, 1);
684 if (ret)
685 goto del_hash;
686 clip_valid = true;
687 }
688 ret = cxgb4_create_server6(ndev, stid,
689 &sk->sk_v6_rcv_saddr,
690 inet_sk(sk)->inet_sport,
691 cdev->lldi->rxq_ids[0]);
692 }
cc35c88a
AG		 693 if (ret > 0)
694 ret = net_xmit_errno(ret);
695 if (ret)
696 goto del_hash;
697 return 0;
698del_hash:
6abde0b2
VKY		 699 if (clip_valid)
700 cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1);
cc35c88a
AG		 701 listen_hash_del(cdev, sk);
702free_stid:
703 cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
704 sock_put(sk);
705free_ctx:
706 kfree(ctx);
707 module_put(THIS_MODULE);
708 return -EBADF;
709}
710
711void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
712{
713 struct listen_ctx *listen_ctx;
6abde0b2
VKY		 714 struct chtls_sock *csk;
715 int addr_type = 0;
cc35c88a
AG		 716 int stid;
717
718 stid = listen_hash_del(cdev, sk);
719 if (stid < 0)
720 return;
721
722 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
723 chtls_reset_synq(listen_ctx);
724
725 cxgb4_remove_server(cdev->lldi->ports[0], stid,
6abde0b2
VKY		 726 cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6);
727
728 if (sk->sk_family == PF_INET6) {
729 csk = rcu_dereference_sk_user_data(sk);
730 addr_type = ipv6_addr_type((const struct in6_addr *)
731 &sk->sk_v6_rcv_saddr);
732 if (addr_type != IPV6_ADDR_ANY)
733 cxgb4_clip_release(csk->egress_dev, (const u32 *)
734 &sk->sk_v6_rcv_saddr, 1);
735 }
cc35c88a
AG		 736 chtls_disconnect_acceptq(sk);
737}
738
739static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
740{
741 struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
742 unsigned int stid = GET_TID(rpl);
743 struct listen_ctx *listen_ctx;
744
745 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
746 if (!listen_ctx)
747 return CPL_RET_BUF_DONE;
748
749 if (listen_ctx->state == T4_LISTEN_START_PENDING) {
750 listen_ctx->state = T4_LISTEN_STARTED;
751 return CPL_RET_BUF_DONE;
752 }
753
754 if (rpl->status != CPL_ERR_NONE) {
755 pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
756 rpl->status, stid);
757 return CPL_RET_BUF_DONE;
758 }
759 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
760 sock_put(listen_ctx->lsk);
761 kfree(listen_ctx);
762 module_put(THIS_MODULE);
763
764 return 0;
765}
766
767static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
768{
769 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
770 struct listen_ctx *listen_ctx;
771 unsigned int stid;
772 void *data;
773
774 stid = GET_TID(rpl);
775 data = lookup_stid(cdev->tids, stid);
776 listen_ctx = (struct listen_ctx *)data;
777
778 if (rpl->status != CPL_ERR_NONE) {
779 pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
780 rpl->status, stid);
781 return CPL_RET_BUF_DONE;
782 }
783
784 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
785 sock_put(listen_ctx->lsk);
786 kfree(listen_ctx);
787 module_put(THIS_MODULE);
788
789 return 0;
790}
791
93e23eb2
VKY		 792static void chtls_purge_wr_queue(struct sock *sk)
793{
794 struct sk_buff *skb;
795
796 while ((skb = dequeue_wr(sk)) != NULL)
797 kfree_skb(skb);
798}
799
cc35c88a
AG		 800static void chtls_release_resources(struct sock *sk)
801{
802 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
803 struct chtls_dev *cdev = csk->cdev;
804 unsigned int tid = csk->tid;
805 struct tid_info *tids;
806
807 if (!cdev)
808 return;
809
810 tids = cdev->tids;
811 kfree_skb(csk->txdata_skb_cache);
812 csk->txdata_skb_cache = NULL;
813
93e23eb2
VKY		 814 if (csk->wr_credits != csk->wr_max_credits) {
815 chtls_purge_wr_queue(sk);
816 chtls_reset_wr_list(csk);
817 }
818
cc35c88a
AG		 819 if (csk->l2t_entry) {
820 cxgb4_l2t_release(csk->l2t_entry);
821 csk->l2t_entry = NULL;
822 }
823
3a0a9783
RM		 824 if (sk->sk_state != TCP_SYN_SENT) {
825 cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
826 sock_put(sk);
827 }
cc35c88a
AG		 828}
829
830static void chtls_conn_done(struct sock *sk)
831{
832 if (sock_flag(sk, SOCK_DEAD))
833 chtls_purge_receive_queue(sk);
834 sk_wakeup_sleepers(sk, 0);
835 tcp_done(sk);
836}
837
838static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
839{
840 /*
841 * If the server is still open we clean up the child connection,
842 * otherwise the server already did the clean up as it was purging
843 * its SYN queue and the skb was just sitting in its backlog.
844 */
845 if (likely(parent->sk_state == TCP_LISTEN)) {
846 cleanup_syn_rcv_conn(child, parent);
847 /* Without the below call to sock_orphan,
848 * we leak the socket resource with syn_flood test
849 * as inet_csk_destroy_sock will not be called
850 * in tcp_done since SOCK_DEAD flag is not set.
851 * Kernel handles this differently where new socket is
852 * created only after 3 way handshake is done.
853 */
854 sock_orphan(child);
855 percpu_counter_inc((child)->sk_prot->orphan_count);
856 chtls_release_resources(child);
857 chtls_conn_done(child);
858 } else {
859 if (csk_flag(child, CSK_RST_ABORTED)) {
860 chtls_release_resources(child);
861 chtls_conn_done(child);
862 }
863 }
864}
865
866static void pass_open_abort(struct sock *child, struct sock *parent,
867 struct sk_buff *skb)
868{
869 do_abort_syn_rcv(child, parent);
870 kfree_skb(skb);
871}
872
873static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
874{
875 pass_open_abort(skb->sk, lsk, skb);
876}
877
878static void chtls_pass_open_arp_failure(struct sock *sk,
879 struct sk_buff *skb)
880{
881 const struct request_sock *oreq;
882 struct chtls_sock *csk;
883 struct chtls_dev *cdev;
884 struct sock *parent;
885 void *data;
886
887 csk = rcu_dereference_sk_user_data(sk);
888 cdev = csk->cdev;
889
890 /*
891 * If the connection is being aborted due to the parent listening
892 * socket going away there's nothing to do, the ABORT_REQ will close
893 * the connection.
894 */
895 if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
896 kfree_skb(skb);
897 return;
898 }
899
900 oreq = csk->passive_reap_next;
901 data = lookup_stid(cdev->tids, oreq->ts_recent);
902 parent = ((struct listen_ctx *)data)->lsk;
903
904 bh_lock_sock(parent);
905 if (!sock_owned_by_user(parent)) {
906 pass_open_abort(sk, parent, skb);
907 } else {
908 BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
909 __sk_add_backlog(parent, skb);
910 }
911 bh_unlock_sock(parent);
912}
913
914static void chtls_accept_rpl_arp_failure(void *handle,
915 struct sk_buff *skb)
916{
917 struct sock *sk = (struct sock *)handle;
918
919 sock_hold(sk);
920 process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
921 sock_put(sk);
922}
923
924static unsigned int chtls_select_mss(const struct chtls_sock *csk,
925 unsigned int pmtu,
926 struct cpl_pass_accept_req *req)
927{
928 struct chtls_dev *cdev;
929 struct dst_entry *dst;
930 unsigned int tcpoptsz;
931 unsigned int iphdrsz;
932 unsigned int mtu_idx;
933 struct tcp_sock *tp;
934 unsigned int mss;
935 struct sock *sk;
936
937 mss = ntohs(req->tcpopt.mss);
938 sk = csk->sk;
939 dst = __sk_dst_get(sk);
940 cdev = csk->cdev;
941 tp = tcp_sk(sk);
942 tcpoptsz = 0;
943
6abde0b2
VKY		 944 if (sk->sk_family == AF_INET6)
945 iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
946 else
947 iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
cc35c88a
AG		 948 if (req->tcpopt.tstamp)
949 tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
950
951 tp->advmss = dst_metric_advmss(dst);
952 if (USER_MSS(tp) && tp->advmss > USER_MSS(tp))
953 tp->advmss = USER_MSS(tp);
954 if (tp->advmss > pmtu - iphdrsz)
955 tp->advmss = pmtu - iphdrsz;
956 if (mss && tp->advmss > mss)
957 tp->advmss = mss;
958
959 tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
960 iphdrsz + tcpoptsz,
961 tp->advmss - tcpoptsz,
962 8, &mtu_idx);
963 tp->advmss -= iphdrsz;
964
965 inet_csk(sk)->icsk_pmtu_cookie = pmtu;
966 return mtu_idx;
967}
968
cc35c88a
AG		 969static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
970{
971 int wscale = 0;
972
973 if (space > MAX_RCV_WND)
974 space = MAX_RCV_WND;
975 if (win_clamp && win_clamp < space)
976 space = win_clamp;
977
978 if (wscale_ok) {
979 while (wscale < 14 && (65535 << wscale) < space)
980 wscale++;
981 }
982 return wscale;
983}
984
985static void chtls_pass_accept_rpl(struct sk_buff *skb,
986 struct cpl_pass_accept_req *req,
987 unsigned int tid)
988
989{
990 struct cpl_t5_pass_accept_rpl *rpl5;
991 struct cxgb4_lld_info *lldi;
992 const struct tcphdr *tcph;
993 const struct tcp_sock *tp;
994 struct chtls_sock *csk;
995 unsigned int len;
996 struct sock *sk;
997 u32 opt2, hlen;
998 u64 opt0;
999
1000 sk = skb->sk;
1001 tp = tcp_sk(sk);
1002 csk = sk->sk_user_data;
1003 csk->tid = tid;
1004 lldi = csk->cdev->lldi;
1005 len = roundup(sizeof(*rpl5), 16);
1006
1007 rpl5 = __skb_put_zero(skb, len);
1008 INIT_TP_WR(rpl5, tid);
1009
1010 OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
1011 csk->tid));
1012 csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
1013 req);
1014 opt0 = TCAM_BYPASS_F |
0c3a16be 1015 WND_SCALE_V(RCV_WSCALE(tp)) |
cc35c88a
AG		 1016 MSS_IDX_V(csk->mtu_idx) |
1017 L2T_IDX_V(csk->l2t_entry->idx) |
1018 NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
1019 TX_CHAN_V(csk->tx_chan) |
1020 SMAC_SEL_V(csk->smac_idx) |
1021 DSCP_V(csk->tos >> 2) |
1022 ULP_MODE_V(ULP_MODE_TLS) |
1023 RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
1024
1025 opt2 = RX_CHANNEL_V(0) |
1026 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
1027
1028 if (!is_t5(lldi->adapter_type))
1029 opt2 |= RX_FC_DISABLE_F;
1030 if (req->tcpopt.tstamp)
1031 opt2 |= TSTAMPS_EN_F;
1032 if (req->tcpopt.sack)
1033 opt2 |= SACK_EN_F;
1034 hlen = ntohl(req->hdr_len);
1035
1036 tcph = (struct tcphdr *)((u8 *)(req + 1) +
1037 T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
1038 if (tcph->ece && tcph->cwr)
1039 opt2 |= CCTRL_ECN_V(1);
1040 opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
1041 opt2 |= T5_ISS_F;
1042 opt2 |= T5_OPT_2_VALID_F;
1043 rpl5->opt0 = cpu_to_be64(opt0);
1044 rpl5->opt2 = cpu_to_be32(opt2);
1045 rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
1046 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
1047 t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
1048 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
1049}
1050
1051static void inet_inherit_port(struct inet_hashinfo *hash_info,
1052 struct sock *lsk, struct sock *newsk)
1053{
1054 local_bh_disable();
1055 __inet_inherit_port(lsk, newsk);
1056 local_bh_enable();
1057}
1058
1059static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1060{
1061 if (skb->protocol) {
1062 kfree_skb(skb);
1063 return 0;
1064 }
1065 BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
1066 return 0;
1067}
1068
0c3a16be
AG		 1069static void chtls_set_tcp_window(struct chtls_sock *csk)
1070{
1071 struct net_device *ndev = csk->egress_dev;
1072 struct port_info *pi = netdev_priv(ndev);
1073 unsigned int linkspeed;
1074 u8 scale;
1075
1076 linkspeed = pi->link_cfg.speed;
1077 scale = linkspeed / SPEED_10000;
1078#define CHTLS_10G_RCVWIN (256 * 1024)
1079 csk->rcv_win = CHTLS_10G_RCVWIN;
1080 if (scale)
1081 csk->rcv_win *= scale;
1082#define CHTLS_10G_SNDWIN (256 * 1024)
1083 csk->snd_win = CHTLS_10G_SNDWIN;
1084 if (scale)
1085 csk->snd_win *= scale;
1086}
1087
cc35c88a
AG		 1088static struct sock *chtls_recv_sock(struct sock *lsk,
1089 struct request_sock *oreq,
1090 void *network_hdr,
1091 const struct cpl_pass_accept_req *req,
1092 struct chtls_dev *cdev)
1093{
cc35c88a
AG		 1094 struct inet_sock *newinet;
1095 const struct iphdr *iph;
76f7164d 1096 struct tls_context *ctx;
cc35c88a
AG		 1097 struct net_device *ndev;
1098 struct chtls_sock *csk;
1099 struct dst_entry *dst;
1100 struct neighbour *n;
1101 struct tcp_sock *tp;
1102 struct sock *newsk;
1103 u16 port_id;
1104 int rxq_idx;
1105 int step;
1106
1107 iph = (const struct iphdr *)network_hdr;
1108 newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
1109 if (!newsk)
1110 goto free_oreq;
1111
6abde0b2
VKY		 1112 if (lsk->sk_family == AF_INET) {
1113 dst = inet_csk_route_child_sock(lsk, newsk, oreq);
1114 if (!dst)
1115 goto free_sk;
cc35c88a 1116
6abde0b2
VKY		 1117 n = dst_neigh_lookup(dst, &iph->saddr);
1118 } else {
1119 const struct ipv6hdr *ip6h;
1120 struct flowi6 fl6;
1121
1122 ip6h = (const struct ipv6hdr *)network_hdr;
1123 memset(&fl6, 0, sizeof(fl6));
1124 fl6.flowi6_proto = IPPROTO_TCP;
1125 fl6.saddr = ip6h->daddr;
1126 fl6.daddr = ip6h->saddr;
1127 fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port;
1128 fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num);
1129 security_req_classify_flow(oreq, flowi6_to_flowi(&fl6));
1130 dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL);
1131 if (IS_ERR(dst))
1132 goto free_sk;
1133 n = dst_neigh_lookup(dst, &ip6h->saddr);
1134 }
cc35c88a
AG		 1135 if (!n)
1136 goto free_sk;
1137
1138 ndev = n->dev;
1139 if (!ndev)
1140 goto free_dst;
1141 port_id = cxgb4_port_idx(ndev);
1142
1143 csk = chtls_sock_create(cdev);
1144 if (!csk)
1145 goto free_dst;
1146
1147 csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
1148 if (!csk->l2t_entry)
1149 goto free_csk;
1150
1151 newsk->sk_user_data = csk;
1152 newsk->sk_backlog_rcv = chtls_backlog_rcv;
1153
1154 tp = tcp_sk(newsk);
1155 newinet = inet_sk(newsk);
1156
6abde0b2
VKY		 1157 if (iph->version == 0x4) {
1158 newinet->inet_daddr = iph->saddr;
1159 newinet->inet_rcv_saddr = iph->daddr;
1160 newinet->inet_saddr = iph->daddr;
1161 } else {
1162 struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk;
1163 struct inet_request_sock *treq = inet_rsk(oreq);
1164 struct ipv6_pinfo *newnp = inet6_sk(newsk);
1165 struct ipv6_pinfo *np = inet6_sk(lsk);
1166
1167 inet_sk(newsk)->pinet6 = &newtcp6sk->inet6;
1168 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1169 newsk->sk_v6_daddr = treq->ir_v6_rmt_addr;
1170 newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr;
1171 inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr;
1172 newnp->ipv6_fl_list = NULL;
1173 newnp->pktoptions = NULL;
1174 newsk->sk_bound_dev_if = treq->ir_iif;
1175 newinet->inet_opt = NULL;
1176 newinet->inet_daddr = LOOPBACK4_IPV6;
1177 newinet->inet_saddr = LOOPBACK4_IPV6;
1178 }
cc35c88a
AG		 1179
1180 oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1181 sk_setup_caps(newsk, dst);
76f7164d
AG		 1182 ctx = tls_get_ctx(lsk);
1183 newsk->sk_destruct = ctx->sk_destruct;
cc35c88a
AG		 1184 csk->sk = newsk;
1185 csk->passive_reap_next = oreq;
1186 csk->tx_chan = cxgb4_port_chan(ndev);
1187 csk->port_id = port_id;
1188 csk->egress_dev = ndev;
1189 csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
0c3a16be
AG		 1190 chtls_set_tcp_window(csk);
1191 tp->rcv_wnd = csk->rcv_win;
1192 csk->sndbuf = csk->snd_win;
cc35c88a
AG		 1193 csk->ulp_mode = ULP_MODE_TLS;
1194 step = cdev->lldi->nrxq / cdev->lldi->nchan;
1195 csk->rss_qid = cdev->lldi->rxq_ids[port_id * step];
1196 rxq_idx = port_id * step;
1197 csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
1198 port_id * step;
1199 csk->sndbuf = newsk->sk_sndbuf;
02d805dc 1200 csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
cc35c88a 1201 RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
0c3a16be
AG		 1202 sock_net(newsk)->
1203 ipv4.sysctl_tcp_window_scaling,
cc35c88a
AG		 1204 tp->window_clamp);
1205 neigh_release(n);
1206 inet_inherit_port(&tcp_hashinfo, lsk, newsk);
1207 csk_set_flag(csk, CSK_CONN_INLINE);
1208 bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
1209
1210 return newsk;
1211free_csk:
1212 chtls_sock_release(&csk->kref);
1213free_dst:
1214 dst_release(dst);
1215free_sk:
1216 inet_csk_prepare_forced_close(newsk);
1217 tcp_done(newsk);
1218free_oreq:
1219 chtls_reqsk_free(oreq);
1220 return NULL;
1221}
1222
1223/*
1224 * Populate a TID_RELEASE WR. The skb must be already propely sized.
1225 */
1226static void mk_tid_release(struct sk_buff *skb,
1227 unsigned int chan, unsigned int tid)
1228{
1229 struct cpl_tid_release *req;
1230 unsigned int len;
1231
1232 len = roundup(sizeof(struct cpl_tid_release), 16);
1233 req = (struct cpl_tid_release *)__skb_put(skb, len);
1234 memset(req, 0, len);
1235 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1236 INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
1237}
1238
1239static int chtls_get_module(struct sock *sk)
1240{
1241 struct inet_connection_sock *icsk = inet_csk(sk);
1242
1243 if (!try_module_get(icsk->icsk_ulp_ops->owner))
1244 return -1;
1245
1246 return 0;
1247}
1248
1249static void chtls_pass_accept_request(struct sock *sk,
1250 struct sk_buff *skb)
1251{
1252 struct cpl_t5_pass_accept_rpl *rpl;
1253 struct cpl_pass_accept_req *req;
1254 struct listen_ctx *listen_ctx;
0c3a16be 1255 struct vlan_ethhdr *vlan_eh;
cc35c88a
AG		 1256 struct request_sock *oreq;
1257 struct sk_buff *reply_skb;
1258 struct chtls_sock *csk;
1259 struct chtls_dev *cdev;
6abde0b2 1260 struct ipv6hdr *ip6h;
cc35c88a
AG		 1261 struct tcphdr *tcph;
1262 struct sock *newsk;
1263 struct ethhdr *eh;
1264 struct iphdr *iph;
1265 void *network_hdr;
1266 unsigned int stid;
1267 unsigned int len;
1268 unsigned int tid;
0c3a16be
AG		 1269 bool th_ecn, ect;
1270 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
1271 u16 eth_hdr_len;
1272 bool ecn_ok;
cc35c88a
AG		 1273
1274 req = cplhdr(skb) + RSS_HDR;
1275 tid = GET_TID(req);
1276 cdev = BLOG_SKB_CB(skb)->cdev;
1277 newsk = lookup_tid(cdev->tids, tid);
1278 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1279 if (newsk) {
1280 pr_info("tid (%d) already in use\n", tid);
1281 return;
1282 }
1283
1284 len = roundup(sizeof(*rpl), 16);
1285 reply_skb = alloc_skb(len, GFP_ATOMIC);
1286 if (!reply_skb) {
1287 cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
1288 kfree_skb(skb);
1289 return;
1290 }
1291
1292 if (sk->sk_state != TCP_LISTEN)
1293 goto reject;
1294
1295 if (inet_csk_reqsk_queue_is_full(sk))
1296 goto reject;
1297
1298 if (sk_acceptq_is_full(sk))
1299 goto reject;
1300
cc35c88a 1301
0c3a16be
AG		 1302 eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));
1303 if (eth_hdr_len == ETH_HLEN) {
1304 eh = (struct ethhdr *)(req + 1);
1305 iph = (struct iphdr *)(eh + 1);
6abde0b2 1306 ip6h = (struct ipv6hdr *)(eh + 1);
0c3a16be
AG		 1307 network_hdr = (void *)(eh + 1);
1308 } else {
1309 vlan_eh = (struct vlan_ethhdr *)(req + 1);
1310 iph = (struct iphdr *)(vlan_eh + 1);
6abde0b2 1311 ip6h = (struct ipv6hdr *)(vlan_eh + 1);
0c3a16be
AG		 1312 network_hdr = (void *)(vlan_eh + 1);
1313 }
cc35c88a 1314
6abde0b2
VKY		 1315 if (iph->version == 0x4) {
1316 tcph = (struct tcphdr *)(iph + 1);
1317 skb_set_network_header(skb, (void *)iph - (void *)req);
1318 oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
1319 } else {
1320 tcph = (struct tcphdr *)(ip6h + 1);
1321 skb_set_network_header(skb, (void *)ip6h - (void *)req);
1322 oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false);
1323 }
1324
1325 if (!oreq)
1326 goto reject;
1327
1328 oreq->rsk_rcv_wnd = 0;
1329 oreq->rsk_window_clamp = 0;
1330 oreq->cookie_ts = 0;
1331 oreq->mss = 0;
1332 oreq->ts_recent = 0;
cc35c88a
AG		 1333
1334 tcp_rsk(oreq)->tfo_listener = false;
1335 tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
1336 chtls_set_req_port(oreq, tcph->source, tcph->dest);
6abde0b2
VKY		 1337 if (iph->version == 0x4) {
1338 chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
1339 ip_dsfield = ipv4_get_dsfield(iph);
1340 } else {
1341 inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
1342 inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
1343 ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb));
1344 }
0c3a16be
AG		 1345 if (req->tcpopt.wsf <= 14 &&
1346 sock_net(sk)->ipv4.sysctl_tcp_window_scaling) {
cc35c88a
AG		 1347 inet_rsk(oreq)->wscale_ok = 1;
1348 inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
1349 }
1350 inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
0c3a16be
AG		 1351 th_ecn = tcph->ece && tcph->cwr;
1352 if (th_ecn) {
1353 ect = !INET_ECN_is_not_ect(ip_dsfield);
1354 ecn_ok = sock_net(sk)->ipv4.sysctl_tcp_ecn;
1355 if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))
1356 inet_rsk(oreq)->ecn_ok = 1;
1357 }
cc35c88a
AG		 1358
1359 newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
1360 if (!newsk)
6abde0b2 1361 goto free_oreq;
cc35c88a
AG		 1362
1363 if (chtls_get_module(newsk))
1364 goto reject;
1365 inet_csk_reqsk_queue_added(sk);
1366 reply_skb->sk = newsk;
1367 chtls_install_cpl_ops(newsk);
1368 cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
1369 csk = rcu_dereference_sk_user_data(newsk);
1370 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
1371 csk->listen_ctx = listen_ctx;
1372 __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
1373 chtls_pass_accept_rpl(reply_skb, req, tid);
1374 kfree_skb(skb);
1375 return;
1376
1377free_oreq:
1378 chtls_reqsk_free(oreq);
1379reject:
1380 mk_tid_release(reply_skb, 0, tid);
1381 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1382 kfree_skb(skb);
1383}
1384
1385/*
1386 * Handle a CPL_PASS_ACCEPT_REQ message.
1387 */
1388static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
1389{
1390 struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
1391 struct listen_ctx *ctx;
1392 unsigned int stid;
1393 unsigned int tid;
1394 struct sock *lsk;
1395 void *data;
1396
1397 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1398 tid = GET_TID(req);
1399
1400 data = lookup_stid(cdev->tids, stid);
1401 if (!data)
1402 return 1;
1403
1404 ctx = (struct listen_ctx *)data;
1405 lsk = ctx->lsk;
1406
59437d78 1407 if (unlikely(tid_out_of_range(cdev->tids, tid))) {
cc35c88a
AG		 1408 pr_info("passive open TID %u too large\n", tid);
1409 return 1;
1410 }
1411
1412 BLOG_SKB_CB(skb)->cdev = cdev;
1413 process_cpl_msg(chtls_pass_accept_request, lsk, skb);
1414 return 0;
1415}
1416
1417/*
1418 * Completes some final bits of initialization for just established connections
1419 * and changes their state to TCP_ESTABLISHED.
1420 *
1421 * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
1422 */
1423static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
1424{
1425 struct tcp_sock *tp = tcp_sk(sk);
1426
1427 tp->pushed_seq = snd_isn;
1428 tp->write_seq = snd_isn;
1429 tp->snd_nxt = snd_isn;
1430 tp->snd_una = snd_isn;
a904a069 1431 inet_sk(sk)->inet_id = prandom_u32();
cc35c88a
AG		 1432 assign_rxopt(sk, opt);
1433
1434 if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
1435 tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
1436
1437 smp_mb();
1438 tcp_set_state(sk, TCP_ESTABLISHED);
1439}
1440
1441static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
1442{
1443 struct sk_buff *abort_skb;
1444
1445 abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
1446 if (abort_skb)
1447 chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
1448}
1449
1450static struct sock *reap_list;
1451static DEFINE_SPINLOCK(reap_list_lock);
1452
1453/*
1454 * Process the reap list.
1455 */
1456DECLARE_TASK_FUNC(process_reap_list, task_param)
1457{
1458 spin_lock_bh(&reap_list_lock);
1459 while (reap_list) {
1460 struct sock *sk = reap_list;
1461 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1462
1463 reap_list = csk->passive_reap_next;
1464 csk->passive_reap_next = NULL;
1465 spin_unlock(&reap_list_lock);
1466 sock_hold(sk);
1467
1468 bh_lock_sock(sk);
1469 chtls_abort_conn(sk, NULL);
1470 sock_orphan(sk);
1471 if (sk->sk_state == TCP_CLOSE)
1472 inet_csk_destroy_sock(sk);
1473 bh_unlock_sock(sk);
1474 sock_put(sk);
1475 spin_lock(&reap_list_lock);
1476 }
1477 spin_unlock_bh(&reap_list_lock);
1478}
1479
1480static DECLARE_WORK(reap_task, process_reap_list);
1481
1482static void add_to_reap_list(struct sock *sk)
1483{
1484 struct chtls_sock *csk = sk->sk_user_data;
1485
1486 local_bh_disable();
1487 bh_lock_sock(sk);
1488 release_tcp_port(sk); /* release the port immediately */
1489
1490 spin_lock(&reap_list_lock);
1491 csk->passive_reap_next = reap_list;
1492 reap_list = sk;
1493 if (!csk->passive_reap_next)
1494 schedule_work(&reap_task);
1495 spin_unlock(&reap_list_lock);
1496 bh_unlock_sock(sk);
1497 local_bh_enable();
1498}
1499
1500static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
1501 struct chtls_dev *cdev)
1502{
1503 struct request_sock *oreq;
1504 struct chtls_sock *csk;
1505
1506 if (lsk->sk_state != TCP_LISTEN)
1507 return;
1508
1509 csk = child->sk_user_data;
1510 oreq = csk->passive_reap_next;
1511 csk->passive_reap_next = NULL;
1512
1513 reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
1514 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
1515
1516 if (sk_acceptq_is_full(lsk)) {
1517 chtls_reqsk_free(oreq);
1518 add_to_reap_list(child);
1519 } else {
1520 refcount_set(&oreq->rsk_refcnt, 1);
1521 inet_csk_reqsk_queue_add(lsk, oreq, child);
1522 lsk->sk_data_ready(lsk);
1523 }
1524}
1525
1526static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
1527{
1528 struct sock *child = skb->sk;
1529
1530 skb->sk = NULL;
1531 add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
1532 kfree_skb(skb);
1533}
1534
1535static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
1536{
1537 struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
1538 struct chtls_sock *csk;
1539 struct sock *lsk, *sk;
1540 unsigned int hwtid;
1541
1542 hwtid = GET_TID(req);
1543 sk = lookup_tid(cdev->tids, hwtid);
1544 if (!sk)
1545 return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
1546
1547 bh_lock_sock(sk);
1548 if (unlikely(sock_owned_by_user(sk))) {
1549 kfree_skb(skb);
1550 } else {
1551 unsigned int stid;
1552 void *data;
1553
1554 csk = sk->sk_user_data;
1555 csk->wr_max_credits = 64;
1556 csk->wr_credits = 64;
1557 csk->wr_unacked = 0;
1558 make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
1559 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1560 sk->sk_state_change(sk);
1561 if (unlikely(sk->sk_socket))
1562 sk_wake_async(sk, 0, POLL_OUT);
1563
1564 data = lookup_stid(cdev->tids, stid);
1565 lsk = ((struct listen_ctx *)data)->lsk;
1566
1567 bh_lock_sock(lsk);
1568 if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
1569 /* removed from synq */
1570 bh_unlock_sock(lsk);
1571 kfree_skb(skb);
1572 goto unlock;
1573 }
1574
1575 if (likely(!sock_owned_by_user(lsk))) {
1576 kfree_skb(skb);
1577 add_pass_open_to_parent(sk, lsk, cdev);
1578 } else {
1579 skb->sk = sk;
1580 BLOG_SKB_CB(skb)->cdev = cdev;
1581 BLOG_SKB_CB(skb)->backlog_rcv =
1582 bl_add_pass_open_to_parent;
1583 __sk_add_backlog(lsk, skb);
1584 }
1585 bh_unlock_sock(lsk);
1586 }
1587unlock:
1588 bh_unlock_sock(sk);
1589 return 0;
1590}
1591
1592/*
1593 * Handle receipt of an urgent pointer.
1594 */
1595static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
1596{
1597 struct tcp_sock *tp = tcp_sk(sk);
1598
1599 urg_seq--;
1600 if (tp->urg_data && !after(urg_seq, tp->urg_seq))
1601 return; /* duplicate pointer */
1602
1603 sk_send_sigurg(sk);
1604 if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
1605 !sock_flag(sk, SOCK_URGINLINE) &&
1606 tp->copied_seq != tp->rcv_nxt) {
1607 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1608
1609 tp->copied_seq++;
1610 if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
1611 chtls_free_skb(sk, skb);
1612 }
1613
1614 tp->urg_data = TCP_URG_NOTYET;
1615 tp->urg_seq = urg_seq;
1616}
1617
1618static void check_sk_callbacks(struct chtls_sock *csk)
1619{
1620 struct sock *sk = csk->sk;
1621
1622 if (unlikely(sk->sk_user_data &&
1623 !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
1624 csk_set_flag(csk, CSK_CALLBACKS_CHKD);
1625}
1626
1627/*
1628 * Handles Rx data that arrives in a state where the socket isn't accepting
1629 * new data.
1630 */
1631static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
1632{
1633 if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
1634 chtls_abort_conn(sk, skb);
1635
1636 kfree_skb(skb);
1637}
1638
1639static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
1640{
1641 struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
1642 struct chtls_sock *csk;
1643 struct tcp_sock *tp;
1644
1645 csk = rcu_dereference_sk_user_data(sk);
1646 tp = tcp_sk(sk);
1647
1648 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1649 handle_excess_rx(sk, skb);
1650 return;
1651 }
1652
1653 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1654 ULP_SKB_CB(skb)->psh = hdr->psh;
1655 skb_ulp_mode(skb) = ULP_MODE_NONE;
1656
1657 skb_reset_transport_header(skb);
1658 __skb_pull(skb, sizeof(*hdr) + RSS_HDR);
1659 if (!skb->data_len)
1660 __skb_trim(skb, ntohs(hdr->len));
1661
1662 if (unlikely(hdr->urg))
1663 handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
1664 if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
1665 tp->urg_seq - tp->rcv_nxt < skb->len))
1666 tp->urg_data = TCP_URG_VALID |
1667 skb->data[tp->urg_seq - tp->rcv_nxt];
1668
1669 if (unlikely(hdr->dack_mode != csk->delack_mode)) {
1670 csk->delack_mode = hdr->dack_mode;
1671 csk->delack_seq = tp->rcv_nxt;
1672 }
1673
1674 tcp_hdr(skb)->fin = 0;
1675 tp->rcv_nxt += skb->len;
1676
1677 __skb_queue_tail(&sk->sk_receive_queue, skb);
1678
1679 if (!sock_flag(sk, SOCK_DEAD)) {
1680 check_sk_callbacks(csk);
1681 sk->sk_data_ready(sk);
1682 }
1683}
1684
1685static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
1686{
1687 struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
1688 unsigned int hwtid = GET_TID(req);
1689 struct sock *sk;
1690
1691 sk = lookup_tid(cdev->tids, hwtid);
3d8ccf9f
GS		 1692 if (unlikely(!sk)) {
1693 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1694 return -EINVAL;
1695 }
cc35c88a
AG		 1696 skb_dst_set(skb, NULL);
1697 process_cpl_msg(chtls_recv_data, sk, skb);
1698 return 0;
1699}
1700
1701static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
1702{
1703 struct cpl_tls_data *hdr = cplhdr(skb);
1704 struct chtls_sock *csk;
1705 struct chtls_hws *tlsk;
1706 struct tcp_sock *tp;
1707
1708 csk = rcu_dereference_sk_user_data(sk);
1709 tlsk = &csk->tlshws;
1710 tp = tcp_sk(sk);
1711
1712 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1713 handle_excess_rx(sk, skb);
1714 return;
1715 }
1716
1717 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1718 ULP_SKB_CB(skb)->flags = 0;
1719 skb_ulp_mode(skb) = ULP_MODE_TLS;
1720
1721 skb_reset_transport_header(skb);
1722 __skb_pull(skb, sizeof(*hdr));
1723 if (!skb->data_len)
1724 __skb_trim(skb,
1725 CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
1726
1727 if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
1728 tp->rcv_nxt < skb->len))
1729 tp->urg_data = TCP_URG_VALID |
1730 skb->data[tp->urg_seq - tp->rcv_nxt];
1731
1732 tcp_hdr(skb)->fin = 0;
1733 tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
1734 __skb_queue_tail(&tlsk->sk_recv_queue, skb);
1735}
1736
1737static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
1738{
1739 struct cpl_tls_data *req = cplhdr(skb);
1740 unsigned int hwtid = GET_TID(req);
1741 struct sock *sk;
1742
1743 sk = lookup_tid(cdev->tids, hwtid);
3d8ccf9f
GS		 1744 if (unlikely(!sk)) {
1745 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1746 return -EINVAL;
1747 }
cc35c88a
AG		 1748 skb_dst_set(skb, NULL);
1749 process_cpl_msg(chtls_recv_pdu, sk, skb);
1750 return 0;
1751}
1752
1753static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
1754{
1755 struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
1756
1757 skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
1758 tls_cmp_hdr->length = ntohs((__force __be16)nlen);
1759}
1760
1761static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
1762{
17a7d24a
AG		 1763 struct tlsrx_cmp_hdr *tls_hdr_pkt;
1764 struct cpl_rx_tls_cmp *cmp_cpl;
cc35c88a
AG		 1765 struct sk_buff *skb_rec;
1766 struct chtls_sock *csk;
1767 struct chtls_hws *tlsk;
1768 struct tcp_sock *tp;
1769
17a7d24a 1770 cmp_cpl = cplhdr(skb);
cc35c88a
AG		 1771 csk = rcu_dereference_sk_user_data(sk);
1772 tlsk = &csk->tlshws;
1773 tp = tcp_sk(sk);
1774
1775 ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
1776 ULP_SKB_CB(skb)->flags = 0;
1777
1778 skb_reset_transport_header(skb);
1779 __skb_pull(skb, sizeof(*cmp_cpl));
17a7d24a
AG		 1780 tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
1781 if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
1782 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
cc35c88a 1783 if (!skb->data_len)
17a7d24a 1784 __skb_trim(skb, TLS_HEADER_LENGTH);
cc35c88a
AG		 1785
1786 tp->rcv_nxt +=
1787 CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
1788
17a7d24a 1789 ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
cc35c88a
AG		 1790 skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
1791 if (!skb_rec) {
cc35c88a
AG		 1792 __skb_queue_tail(&sk->sk_receive_queue, skb);
1793 } else {
1794 chtls_set_hdrlen(skb, tlsk->pldlen);
1795 tlsk->pldlen = 0;
1796 __skb_queue_tail(&sk->sk_receive_queue, skb);
1797 __skb_queue_tail(&sk->sk_receive_queue, skb_rec);
1798 }
1799
1800 if (!sock_flag(sk, SOCK_DEAD)) {
1801 check_sk_callbacks(csk);
1802 sk->sk_data_ready(sk);
1803 }
1804}
1805
1806static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
1807{
1808 struct cpl_rx_tls_cmp *req = cplhdr(skb);
1809 unsigned int hwtid = GET_TID(req);
1810 struct sock *sk;
1811
1812 sk = lookup_tid(cdev->tids, hwtid);
3d8ccf9f
GS		 1813 if (unlikely(!sk)) {
1814 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1815 return -EINVAL;
1816 }
cc35c88a
AG		 1817 skb_dst_set(skb, NULL);
1818 process_cpl_msg(chtls_rx_hdr, sk, skb);
1819
1820 return 0;
1821}
1822
1823static void chtls_timewait(struct sock *sk)
1824{
1825 struct tcp_sock *tp = tcp_sk(sk);
1826
1827 tp->rcv_nxt++;
cca9bab1 1828 tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
cc35c88a
AG		 1829 tp->srtt_us = 0;
1830 tcp_time_wait(sk, TCP_TIME_WAIT, 0);
1831}
1832
1833static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
1834{
1835 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1836
3a0a9783
RM		 1837 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1838 goto out;
1839
cc35c88a
AG		 1840 sk->sk_shutdown |= RCV_SHUTDOWN;
1841 sock_set_flag(sk, SOCK_DONE);
1842
1843 switch (sk->sk_state) {
1844 case TCP_SYN_RECV:
1845 case TCP_ESTABLISHED:
1846 tcp_set_state(sk, TCP_CLOSE_WAIT);
1847 break;
1848 case TCP_FIN_WAIT1:
1849 tcp_set_state(sk, TCP_CLOSING);
1850 break;
1851 case TCP_FIN_WAIT2:
1852 chtls_release_resources(sk);
1853 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1854 chtls_conn_done(sk);
1855 else
1856 chtls_timewait(sk);
1857 break;
1858 default:
1859 pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
1860 }
1861
1862 if (!sock_flag(sk, SOCK_DEAD)) {
1863 sk->sk_state_change(sk);
1864 /* Do not send POLL_HUP for half duplex close. */
1865
1866 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1867 sk->sk_state == TCP_CLOSE)
1868 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
1869 else
1870 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
1871 }
3a0a9783 1872out:
93e23eb2 1873 kfree_skb(skb);
cc35c88a
AG		 1874}
1875
1876static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
1877{
1878 struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
1879 struct chtls_sock *csk;
1880 struct tcp_sock *tp;
1881
1882 csk = rcu_dereference_sk_user_data(sk);
3a0a9783
RM		 1883
1884 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1885 goto out;
1886
cc35c88a
AG		 1887 tp = tcp_sk(sk);
1888
1889 tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
1890
1891 switch (sk->sk_state) {
1892 case TCP_CLOSING:
1893 chtls_release_resources(sk);
1894 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1895 chtls_conn_done(sk);
1896 else
1897 chtls_timewait(sk);
1898 break;
1899 case TCP_LAST_ACK:
1900 chtls_release_resources(sk);
1901 chtls_conn_done(sk);
1902 break;
1903 case TCP_FIN_WAIT1:
1904 tcp_set_state(sk, TCP_FIN_WAIT2);
1905 sk->sk_shutdown |= SEND_SHUTDOWN;
1906
1907 if (!sock_flag(sk, SOCK_DEAD))
1908 sk->sk_state_change(sk);
1909 else if (tcp_sk(sk)->linger2 < 0 &&
1910 !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
1911 chtls_abort_conn(sk, skb);
1912 break;
1913 default:
1914 pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
1915 }
3a0a9783 1916out:
cc35c88a
AG		 1917 kfree_skb(skb);
1918}
1919
1920static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
1921 size_t len, gfp_t gfp)
1922{
1923 if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
1924 WARN_ONCE(skb->len < len, "skb alloc error");
1925 __skb_trim(skb, len);
1926 skb_get(skb);
1927 } else {
1928 skb = alloc_skb(len, gfp);
1929 if (skb)
1930 __skb_put(skb, len);
1931 }
1932 return skb;
1933}
1934
1935static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
1936 int cmd)
1937{
1938 struct cpl_abort_rpl *rpl = cplhdr(skb);
1939
1940 INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
1941 rpl->cmd = cmd;
1942}
1943
1944static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
1945{
1946 struct cpl_abort_req_rss *req = cplhdr(skb);
1947 struct sk_buff *reply_skb;
1948
1949 reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1950 GFP_KERNEL | __GFP_NOFAIL);
1951 __skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
1952 set_abort_rpl_wr(reply_skb, GET_TID(req),
1953 (req->status & CPL_ABORT_NO_RST));
1954 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
1955 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1956 kfree_skb(skb);
1957}
1958
c9f0d33c
VKY		 1959/*
1960 * Add an skb to the deferred skb queue for processing from process context.
1961 */
1962static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
1963 defer_handler_t handler)
1964{
1965 DEFERRED_SKB_CB(skb)->handler = handler;
1966 spin_lock_bh(&cdev->deferq.lock);
1967 __skb_queue_tail(&cdev->deferq, skb);
1968 if (skb_queue_len(&cdev->deferq) == 1)
1969 schedule_work(&cdev->deferq_task);
1970 spin_unlock_bh(&cdev->deferq.lock);
1971}
1972
cc35c88a
AG		 1973static void send_abort_rpl(struct sock *sk, struct sk_buff *skb,
1974 struct chtls_dev *cdev, int status, int queue)
1975{
1976 struct cpl_abort_req_rss *req = cplhdr(skb);
1977 struct sk_buff *reply_skb;
1978 struct chtls_sock *csk;
1979
1980 csk = rcu_dereference_sk_user_data(sk);
1981
1982 reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1983 GFP_KERNEL);
1984
1985 if (!reply_skb) {
1986 req->status = (queue << 1);
c9f0d33c 1987 t4_defer_reply(skb, cdev, send_defer_abort_rpl);
cc35c88a
AG		 1988 return;
1989 }
1990
1991 set_abort_rpl_wr(reply_skb, GET_TID(req), status);
1992 kfree_skb(skb);
1993
1994 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
1995 if (csk_conn_inline(csk)) {
1996 struct l2t_entry *e = csk->l2t_entry;
1997
1998 if (e && sk->sk_state != TCP_SYN_RECV) {
1999 cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
2000 return;
2001 }
2002 }
2003 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
2004}
2005
cc35c88a
AG		 2006static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
2007 struct chtls_dev *cdev,
2008 int status, int queue)
2009{
2010 struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
2011 struct sk_buff *reply_skb;
2012 struct chtls_sock *csk;
2013 unsigned int tid;
2014
2015 csk = rcu_dereference_sk_user_data(sk);
2016 tid = GET_TID(req);
2017
2018 reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
2019 if (!reply_skb) {
2020 req->status = (queue << 1) | status;
2021 t4_defer_reply(skb, cdev, send_defer_abort_rpl);
2022 return;
2023 }
2024
2025 set_abort_rpl_wr(reply_skb, tid, status);
3a0a9783 2026 kfree_skb(skb);
cc35c88a
AG		 2027 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
2028 if (csk_conn_inline(csk)) {
2029 struct l2t_entry *e = csk->l2t_entry;
2030
2031 if (e && sk->sk_state != TCP_SYN_RECV) {
2032 cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
2033 return;
2034 }
2035 }
2036 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
cc35c88a
AG		 2037}
2038
2039/*
2040 * This is run from a listener's backlog to abort a child connection in
2041 * SYN_RCV state (i.e., one on the listener's SYN queue).
2042 */
2043static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
2044{
2045 struct chtls_sock *csk;
2046 struct sock *child;
2047 int queue;
2048
2049 child = skb->sk;
2050 csk = rcu_dereference_sk_user_data(child);
2051 queue = csk->txq_idx;
2052
2053 skb->sk = NULL;
2054 do_abort_syn_rcv(child, lsk);
2055 send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
2056 CPL_ABORT_NO_RST, queue);
2057}
2058
2059static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
2060{
2061 const struct request_sock *oreq;
2062 struct listen_ctx *listen_ctx;
2063 struct chtls_sock *csk;
2064 struct chtls_dev *cdev;
2065 struct sock *psk;
2066 void *ctx;
2067
2068 csk = sk->sk_user_data;
2069 oreq = csk->passive_reap_next;
2070 cdev = csk->cdev;
2071
2072 if (!oreq)
2073 return -1;
2074
2075 ctx = lookup_stid(cdev->tids, oreq->ts_recent);
2076 if (!ctx)
2077 return -1;
2078
2079 listen_ctx = (struct listen_ctx *)ctx;
2080 psk = listen_ctx->lsk;
2081
2082 bh_lock_sock(psk);
2083 if (!sock_owned_by_user(psk)) {
2084 int queue = csk->txq_idx;
2085
2086 do_abort_syn_rcv(sk, psk);
2087 send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
2088 } else {
2089 skb->sk = sk;
2090 BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
2091 __sk_add_backlog(psk, skb);
2092 }
2093 bh_unlock_sock(psk);
2094 return 0;
2095}
2096
2097static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
2098{
2099 const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
2100 struct chtls_sock *csk = sk->sk_user_data;
2101 int rst_status = CPL_ABORT_NO_RST;
2102 int queue = csk->txq_idx;
2103
2104 if (is_neg_adv(req->status)) {
2105 if (sk->sk_state == TCP_SYN_RECV)
2106 chtls_set_tcb_tflag(sk, 0, 0);
2107
2108 kfree_skb(skb);
2109 return;
2110 }
2111
2112 csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
2113
2114 if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
2115 !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
2116 struct tcp_sock *tp = tcp_sk(sk);
2117
2118 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
2119 WARN_ONCE(1, "send_tx_flowc error");
2120 csk_set_flag(csk, CSK_TX_DATA_SENT);
2121 }
2122
2123 csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
2124
2125 if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2126 sk->sk_err = ETIMEDOUT;
2127
2128 if (!sock_flag(sk, SOCK_DEAD))
2129 sk->sk_error_report(sk);
2130
2131 if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
2132 return;
2133
2134 chtls_release_resources(sk);
2135 chtls_conn_done(sk);
2136 }
2137
3a0a9783
RM		 2138 chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev,
2139 rst_status, queue);
cc35c88a
AG		 2140}
2141
2142static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
2143{
2144 struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
2145 struct chtls_sock *csk;
2146 struct chtls_dev *cdev;
2147
2148 csk = rcu_dereference_sk_user_data(sk);
2149 cdev = csk->cdev;
2150
2151 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2152 csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
2153 if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
2154 if (sk->sk_state == TCP_SYN_SENT) {
2155 cxgb4_remove_tid(cdev->tids,
2156 csk->port_id,
2157 GET_TID(rpl),
2158 sk->sk_family);
2159 sock_put(sk);
2160 }
2161 chtls_release_resources(sk);
2162 chtls_conn_done(sk);
2163 }
2164 }
2165 kfree_skb(skb);
2166}
2167
2168static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
2169{
2170 struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
2171 void (*fn)(struct sock *sk, struct sk_buff *skb);
2172 unsigned int hwtid = GET_TID(req);
3a0a9783 2173 struct chtls_sock *csk;
cc35c88a
AG		 2174 struct sock *sk;
2175 u8 opcode;
2176
2177 opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
2178
2179 sk = lookup_tid(cdev->tids, hwtid);
2180 if (!sk)
2181 goto rel_skb;
2182
3a0a9783
RM		 2183 csk = sk->sk_user_data;
2184
cc35c88a
AG		 2185 switch (opcode) {
2186 case CPL_PEER_CLOSE:
2187 fn = chtls_peer_close;
2188 break;
2189 case CPL_CLOSE_CON_RPL:
2190 fn = chtls_close_con_rpl;
2191 break;
2192 case CPL_ABORT_REQ_RSS:
3a0a9783
RM		 2193 /*
2194 * Save the offload device in the skb, we may process this
2195 * message after the socket has closed.
2196 */
2197 BLOG_SKB_CB(skb)->cdev = csk->cdev;
cc35c88a
AG		 2198 fn = chtls_abort_req_rss;
2199 break;
2200 case CPL_ABORT_RPL_RSS:
2201 fn = chtls_abort_rpl_rss;
2202 break;
2203 default:
2204 goto rel_skb;
2205 }
2206
2207 process_cpl_msg(fn, sk, skb);
2208 return 0;
2209
2210rel_skb:
2211 kfree_skb(skb);
2212 return 0;
2213}
2214
cc35c88a
AG		 2215static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
2216{
2217 struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
2218 struct chtls_sock *csk = sk->sk_user_data;
2219 struct tcp_sock *tp = tcp_sk(sk);
2220 u32 credits = hdr->credits;
2221 u32 snd_una;
2222
2223 snd_una = ntohl(hdr->snd_una);
2224 csk->wr_credits += credits;
2225
2226 if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
2227 csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
2228
2229 while (credits) {
2230 struct sk_buff *pskb = csk->wr_skb_head;
2231 u32 csum;
2232
2233 if (unlikely(!pskb)) {
2234 if (csk->wr_nondata)
2235 csk->wr_nondata -= credits;
2236 break;
2237 }
2238 csum = (__force u32)pskb->csum;
2239 if (unlikely(credits < csum)) {
2240 pskb->csum = (__force __wsum)(csum - credits);
2241 break;
2242 }
2243 dequeue_wr(sk);
2244 credits -= csum;
2245 kfree_skb(pskb);
2246 }
2247 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
2248 if (unlikely(before(snd_una, tp->snd_una))) {
2249 kfree_skb(skb);
2250 return;
2251 }
2252
2253 if (tp->snd_una != snd_una) {
2254 tp->snd_una = snd_una;
2255 tp->rcv_tstamp = tcp_time_stamp(tp);
2256 if (tp->snd_una == tp->snd_nxt &&
2257 !csk_flag_nochk(csk, CSK_TX_FAILOVER))
2258 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2259 }
2260 }
2261
2262 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
2263 unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
2264
2265 csk->wr_credits -= fclen16;
2266 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2267 csk_reset_flag(csk, CSK_TX_FAILOVER);
2268 }
2269 if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
2270 sk->sk_write_space(sk);
2271
2272 kfree_skb(skb);
2273}
2274
2275static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
2276{
2277 struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
2278 unsigned int hwtid = GET_TID(rpl);
2279 struct sock *sk;
2280
2281 sk = lookup_tid(cdev->tids, hwtid);
3d8ccf9f
GS		 2282 if (unlikely(!sk)) {
2283 pr_err("can't find conn. for hwtid %u.\n", hwtid);
2284 return -EINVAL;
2285 }
cc35c88a
AG		 2286 process_cpl_msg(chtls_rx_ack, sk, skb);
2287
2288 return 0;
2289}
2290
2291chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
2292 [CPL_PASS_OPEN_RPL] = chtls_pass_open_rpl,
2293 [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
2294 [CPL_PASS_ACCEPT_REQ] = chtls_pass_accept_req,
2295 [CPL_PASS_ESTABLISH] = chtls_pass_establish,
2296 [CPL_RX_DATA] = chtls_rx_data,
2297 [CPL_TLS_DATA] = chtls_rx_pdu,
2298 [CPL_RX_TLS_CMP] = chtls_rx_cmp,
2299 [CPL_PEER_CLOSE] = chtls_conn_cpl,
2300 [CPL_CLOSE_CON_RPL] = chtls_conn_cpl,
2301 [CPL_ABORT_REQ_RSS] = chtls_conn_cpl,
2302 [CPL_ABORT_RPL_RSS] = chtls_conn_cpl,
2303 [CPL_FW4_ACK] = chtls_wr_ack,
2304};
Ubuntu Jammy Linux kernel mirror