2 * Copyright (c) 2016 Chelsio Communications, Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 #include <linux/module.h>
10 #include <linux/list.h>
11 #include <linux/workqueue.h>
12 #include <linux/skbuff.h>
13 #include <linux/timer.h>
14 #include <linux/notifier.h>
15 #include <linux/inetdevice.h>
17 #include <linux/tcp.h>
18 #include <linux/if_vlan.h>
20 #include <net/neighbour.h>
21 #include <net/netevent.h>
22 #include <net/route.h>
24 #include <net/ip6_route.h>
25 #include <net/addrconf.h>
27 #include <libcxgb_cm.h>
31 static void cxgbit_init_wr_wait(struct cxgbit_wr_wait
*wr_waitp
)
34 reinit_completion(&wr_waitp
->completion
);
38 cxgbit_wake_up(struct cxgbit_wr_wait
*wr_waitp
, const char *func
, u8 ret
)
40 if (ret
== CPL_ERR_NONE
)
46 pr_err("%s: err:%u", func
, ret
);
48 complete(&wr_waitp
->completion
);
52 cxgbit_wait_for_reply(struct cxgbit_device
*cdev
,
53 struct cxgbit_wr_wait
*wr_waitp
, u32 tid
, u32 timeout
,
58 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
63 ret
= wait_for_completion_timeout(&wr_waitp
->completion
, timeout
* HZ
);
65 pr_info("%s - Device %s not responding tid %u\n",
66 func
, pci_name(cdev
->lldi
.pdev
), tid
);
67 wr_waitp
->ret
= -ETIMEDOUT
;
71 pr_info("%s: FW reply %d tid %u\n",
72 pci_name(cdev
->lldi
.pdev
), wr_waitp
->ret
, tid
);
76 static int cxgbit_np_hashfn(const struct cxgbit_np
*cnp
)
78 return ((unsigned long)cnp
>> 10) & (NP_INFO_HASH_SIZE
- 1);
81 static struct np_info
*
82 cxgbit_np_hash_add(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
,
85 struct np_info
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
88 int bucket
= cxgbit_np_hashfn(cnp
);
92 spin_lock(&cdev
->np_lock
);
93 p
->next
= cdev
->np_hash_tab
[bucket
];
94 cdev
->np_hash_tab
[bucket
] = p
;
95 spin_unlock(&cdev
->np_lock
);
102 cxgbit_np_hash_find(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
104 int stid
= -1, bucket
= cxgbit_np_hashfn(cnp
);
107 spin_lock(&cdev
->np_lock
);
108 for (p
= cdev
->np_hash_tab
[bucket
]; p
; p
= p
->next
) {
114 spin_unlock(&cdev
->np_lock
);
119 static int cxgbit_np_hash_del(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
121 int stid
= -1, bucket
= cxgbit_np_hashfn(cnp
);
122 struct np_info
*p
, **prev
= &cdev
->np_hash_tab
[bucket
];
124 spin_lock(&cdev
->np_lock
);
125 for (p
= *prev
; p
; prev
= &p
->next
, p
= p
->next
) {
133 spin_unlock(&cdev
->np_lock
);
138 void _cxgbit_free_cnp(struct kref
*kref
)
140 struct cxgbit_np
*cnp
;
142 cnp
= container_of(kref
, struct cxgbit_np
, kref
);
147 cxgbit_create_server6(struct cxgbit_device
*cdev
, unsigned int stid
,
148 struct cxgbit_np
*cnp
)
150 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
151 &cnp
->com
.local_addr
;
155 pr_debug("%s: dev = %s; stid = %u; sin6_port = %u\n",
156 __func__
, cdev
->lldi
.ports
[0]->name
, stid
, sin6
->sin6_port
);
158 addr_type
= ipv6_addr_type((const struct in6_addr
*)
160 if (addr_type
!= IPV6_ADDR_ANY
) {
161 ret
= cxgb4_clip_get(cdev
->lldi
.ports
[0],
162 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
164 pr_err("Unable to find clip table entry. laddr %pI6. Error:%d.\n",
165 sin6
->sin6_addr
.s6_addr
, ret
);
171 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
173 ret
= cxgb4_create_server6(cdev
->lldi
.ports
[0],
174 stid
, &sin6
->sin6_addr
,
176 cdev
->lldi
.rxq_ids
[0]);
178 ret
= cxgbit_wait_for_reply(cdev
, &cnp
->com
.wr_wait
,
181 ret
= net_xmit_errno(ret
);
186 if (ret
!= -ETIMEDOUT
)
187 cxgb4_clip_release(cdev
->lldi
.ports
[0],
188 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
190 pr_err("create server6 err %d stid %d laddr %pI6 lport %d\n",
191 ret
, stid
, sin6
->sin6_addr
.s6_addr
,
192 ntohs(sin6
->sin6_port
));
199 cxgbit_create_server4(struct cxgbit_device
*cdev
, unsigned int stid
,
200 struct cxgbit_np
*cnp
)
202 struct sockaddr_in
*sin
= (struct sockaddr_in
*)
203 &cnp
->com
.local_addr
;
206 pr_debug("%s: dev = %s; stid = %u; sin_port = %u\n",
207 __func__
, cdev
->lldi
.ports
[0]->name
, stid
, sin
->sin_port
);
210 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
212 ret
= cxgb4_create_server(cdev
->lldi
.ports
[0],
213 stid
, sin
->sin_addr
.s_addr
,
215 cdev
->lldi
.rxq_ids
[0]);
217 ret
= cxgbit_wait_for_reply(cdev
,
221 ret
= net_xmit_errno(ret
);
226 pr_err("create server failed err %d stid %d laddr %pI4 lport %d\n",
227 ret
, stid
, &sin
->sin_addr
, ntohs(sin
->sin_port
));
231 struct cxgbit_device
*cxgbit_find_device(struct net_device
*ndev
, u8
*port_id
)
233 struct cxgbit_device
*cdev
;
236 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
237 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
239 for (i
= 0; i
< lldi
->nports
; i
++) {
240 if (lldi
->ports
[i
] == ndev
) {
251 static struct net_device
*cxgbit_get_real_dev(struct net_device
*ndev
)
253 if (ndev
->priv_flags
& IFF_BONDING
) {
254 pr_err("Bond devices are not supported. Interface:%s\n",
259 if (is_vlan_dev(ndev
))
260 return vlan_dev_real_dev(ndev
);
265 static struct net_device
*cxgbit_ipv4_netdev(__be32 saddr
)
267 struct net_device
*ndev
;
269 ndev
= __ip_dev_find(&init_net
, saddr
, false);
273 return cxgbit_get_real_dev(ndev
);
276 static struct net_device
*cxgbit_ipv6_netdev(struct in6_addr
*addr6
)
278 struct net_device
*ndev
= NULL
;
281 if (IS_ENABLED(CONFIG_IPV6
)) {
282 for_each_netdev_rcu(&init_net
, ndev
)
283 if (ipv6_chk_addr(&init_net
, addr6
, ndev
, 1)) {
290 return cxgbit_get_real_dev(ndev
);
293 static struct cxgbit_device
*cxgbit_find_np_cdev(struct cxgbit_np
*cnp
)
295 struct sockaddr_storage
*sockaddr
= &cnp
->com
.local_addr
;
296 int ss_family
= sockaddr
->ss_family
;
297 struct net_device
*ndev
= NULL
;
298 struct cxgbit_device
*cdev
= NULL
;
301 if (ss_family
== AF_INET
) {
302 struct sockaddr_in
*sin
;
304 sin
= (struct sockaddr_in
*)sockaddr
;
305 ndev
= cxgbit_ipv4_netdev(sin
->sin_addr
.s_addr
);
306 } else if (ss_family
== AF_INET6
) {
307 struct sockaddr_in6
*sin6
;
309 sin6
= (struct sockaddr_in6
*)sockaddr
;
310 ndev
= cxgbit_ipv6_netdev(&sin6
->sin6_addr
);
315 cdev
= cxgbit_find_device(ndev
, NULL
);
321 static bool cxgbit_inaddr_any(struct cxgbit_np
*cnp
)
323 struct sockaddr_storage
*sockaddr
= &cnp
->com
.local_addr
;
324 int ss_family
= sockaddr
->ss_family
;
327 if (ss_family
== AF_INET
) {
328 struct sockaddr_in
*sin
;
330 sin
= (struct sockaddr_in
*)sockaddr
;
331 if (sin
->sin_addr
.s_addr
== htonl(INADDR_ANY
))
333 } else if (ss_family
== AF_INET6
) {
334 struct sockaddr_in6
*sin6
;
336 sin6
= (struct sockaddr_in6
*)sockaddr
;
337 addr_type
= ipv6_addr_type((const struct in6_addr
*)
339 if (addr_type
== IPV6_ADDR_ANY
)
346 __cxgbit_setup_cdev_np(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
349 int ss_family
= cnp
->com
.local_addr
.ss_family
;
351 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
))
354 stid
= cxgb4_alloc_stid(cdev
->lldi
.tids
, ss_family
, cnp
);
358 if (!cxgbit_np_hash_add(cdev
, cnp
, stid
)) {
359 cxgb4_free_stid(cdev
->lldi
.tids
, stid
, ss_family
);
363 if (ss_family
== AF_INET
)
364 ret
= cxgbit_create_server4(cdev
, stid
, cnp
);
366 ret
= cxgbit_create_server6(cdev
, stid
, cnp
);
369 if (ret
!= -ETIMEDOUT
)
370 cxgb4_free_stid(cdev
->lldi
.tids
, stid
,
372 cxgbit_np_hash_del(cdev
, cnp
);
378 static int cxgbit_setup_cdev_np(struct cxgbit_np
*cnp
)
380 struct cxgbit_device
*cdev
;
383 mutex_lock(&cdev_list_lock
);
384 cdev
= cxgbit_find_np_cdev(cnp
);
388 if (cxgbit_np_hash_find(cdev
, cnp
) >= 0)
391 if (__cxgbit_setup_cdev_np(cdev
, cnp
))
394 cnp
->com
.cdev
= cdev
;
397 mutex_unlock(&cdev_list_lock
);
401 static int cxgbit_setup_all_np(struct cxgbit_np
*cnp
)
403 struct cxgbit_device
*cdev
;
407 mutex_lock(&cdev_list_lock
);
408 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
409 if (cxgbit_np_hash_find(cdev
, cnp
) >= 0) {
410 mutex_unlock(&cdev_list_lock
);
415 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
416 ret
= __cxgbit_setup_cdev_np(cdev
, cnp
);
417 if (ret
== -ETIMEDOUT
)
423 mutex_unlock(&cdev_list_lock
);
425 return count
? 0 : -1;
428 int cxgbit_setup_np(struct iscsi_np
*np
, struct sockaddr_storage
*ksockaddr
)
430 struct cxgbit_np
*cnp
;
433 if ((ksockaddr
->ss_family
!= AF_INET
) &&
434 (ksockaddr
->ss_family
!= AF_INET6
))
437 cnp
= kzalloc(sizeof(*cnp
), GFP_KERNEL
);
441 init_waitqueue_head(&cnp
->accept_wait
);
442 init_completion(&cnp
->com
.wr_wait
.completion
);
443 init_completion(&cnp
->accept_comp
);
444 INIT_LIST_HEAD(&cnp
->np_accept_list
);
445 spin_lock_init(&cnp
->np_accept_lock
);
446 kref_init(&cnp
->kref
);
447 memcpy(&np
->np_sockaddr
, ksockaddr
,
448 sizeof(struct sockaddr_storage
));
449 memcpy(&cnp
->com
.local_addr
, &np
->np_sockaddr
,
450 sizeof(cnp
->com
.local_addr
));
453 cnp
->com
.cdev
= NULL
;
455 if (cxgbit_inaddr_any(cnp
))
456 ret
= cxgbit_setup_all_np(cnp
);
458 ret
= cxgbit_setup_cdev_np(cnp
);
465 np
->np_context
= cnp
;
466 cnp
->com
.state
= CSK_STATE_LISTEN
;
471 cxgbit_set_conn_info(struct iscsi_np
*np
, struct iscsi_conn
*conn
,
472 struct cxgbit_sock
*csk
)
474 conn
->login_family
= np
->np_sockaddr
.ss_family
;
475 conn
->login_sockaddr
= csk
->com
.remote_addr
;
476 conn
->local_sockaddr
= csk
->com
.local_addr
;
479 int cxgbit_accept_np(struct iscsi_np
*np
, struct iscsi_conn
*conn
)
481 struct cxgbit_np
*cnp
= np
->np_context
;
482 struct cxgbit_sock
*csk
;
486 ret
= wait_for_completion_interruptible(&cnp
->accept_comp
);
490 spin_lock_bh(&np
->np_thread_lock
);
491 if (np
->np_thread_state
>= ISCSI_NP_THREAD_RESET
) {
492 spin_unlock_bh(&np
->np_thread_lock
);
494 * No point in stalling here when np_thread
495 * is in state RESET/SHUTDOWN/EXIT - bail
499 spin_unlock_bh(&np
->np_thread_lock
);
501 spin_lock_bh(&cnp
->np_accept_lock
);
502 if (list_empty(&cnp
->np_accept_list
)) {
503 spin_unlock_bh(&cnp
->np_accept_lock
);
507 csk
= list_first_entry(&cnp
->np_accept_list
,
511 list_del_init(&csk
->accept_node
);
512 spin_unlock_bh(&cnp
->np_accept_lock
);
516 cxgbit_set_conn_info(np
, conn
, csk
);
521 __cxgbit_free_cdev_np(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
526 stid
= cxgbit_np_hash_del(cdev
, cnp
);
529 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
))
532 if (cnp
->np
->np_sockaddr
.ss_family
== AF_INET6
)
536 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
537 ret
= cxgb4_remove_server(cdev
->lldi
.ports
[0], stid
,
538 cdev
->lldi
.rxq_ids
[0], ipv6
);
541 ret
= net_xmit_errno(ret
);
548 ret
= cxgbit_wait_for_reply(cdev
, &cnp
->com
.wr_wait
,
550 if (ret
== -ETIMEDOUT
)
553 if (ipv6
&& cnp
->com
.cdev
) {
554 struct sockaddr_in6
*sin6
;
556 sin6
= (struct sockaddr_in6
*)&cnp
->com
.local_addr
;
557 cxgb4_clip_release(cdev
->lldi
.ports
[0],
558 (const u32
*)&sin6
->sin6_addr
.s6_addr
,
562 cxgb4_free_stid(cdev
->lldi
.tids
, stid
,
563 cnp
->com
.local_addr
.ss_family
);
567 static void cxgbit_free_all_np(struct cxgbit_np
*cnp
)
569 struct cxgbit_device
*cdev
;
572 mutex_lock(&cdev_list_lock
);
573 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
574 ret
= __cxgbit_free_cdev_np(cdev
, cnp
);
575 if (ret
== -ETIMEDOUT
)
578 mutex_unlock(&cdev_list_lock
);
581 static void cxgbit_free_cdev_np(struct cxgbit_np
*cnp
)
583 struct cxgbit_device
*cdev
;
586 mutex_lock(&cdev_list_lock
);
587 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
588 if (cdev
== cnp
->com
.cdev
) {
596 __cxgbit_free_cdev_np(cdev
, cnp
);
598 mutex_unlock(&cdev_list_lock
);
601 void cxgbit_free_np(struct iscsi_np
*np
)
603 struct cxgbit_np
*cnp
= np
->np_context
;
605 cnp
->com
.state
= CSK_STATE_DEAD
;
607 cxgbit_free_cdev_np(cnp
);
609 cxgbit_free_all_np(cnp
);
611 np
->np_context
= NULL
;
615 static void cxgbit_send_halfclose(struct cxgbit_sock
*csk
)
618 u32 len
= roundup(sizeof(struct cpl_close_con_req
), 16);
620 skb
= alloc_skb(len
, GFP_ATOMIC
);
624 cxgb_mk_close_con_req(skb
, len
, csk
->tid
, csk
->txq_idx
,
627 cxgbit_skcb_flags(skb
) |= SKCBF_TX_FLAG_COMPL
;
628 __skb_queue_tail(&csk
->txq
, skb
);
629 cxgbit_push_tx_frames(csk
);
632 static void cxgbit_arp_failure_discard(void *handle
, struct sk_buff
*skb
)
634 pr_debug("%s cxgbit_device %p\n", __func__
, handle
);
638 static void cxgbit_abort_arp_failure(void *handle
, struct sk_buff
*skb
)
640 struct cxgbit_device
*cdev
= handle
;
641 struct cpl_abort_req
*req
= cplhdr(skb
);
643 pr_debug("%s cdev %p\n", __func__
, cdev
);
644 req
->cmd
= CPL_ABORT_NO_RST
;
645 cxgbit_ofld_send(cdev
, skb
);
648 static int cxgbit_send_abort_req(struct cxgbit_sock
*csk
)
651 u32 len
= roundup(sizeof(struct cpl_abort_req
), 16);
653 pr_debug("%s: csk %p tid %u; state %d\n",
654 __func__
, csk
, csk
->tid
, csk
->com
.state
);
656 __skb_queue_purge(&csk
->txq
);
658 if (!test_and_set_bit(CSK_TX_DATA_SENT
, &csk
->com
.flags
))
659 cxgbit_send_tx_flowc_wr(csk
);
661 skb
= __skb_dequeue(&csk
->skbq
);
662 cxgb_mk_abort_req(skb
, len
, csk
->tid
, csk
->txq_idx
,
663 csk
->com
.cdev
, cxgbit_abort_arp_failure
);
665 return cxgbit_l2t_send(csk
->com
.cdev
, skb
, csk
->l2t
);
668 void cxgbit_free_conn(struct iscsi_conn
*conn
)
670 struct cxgbit_sock
*csk
= conn
->context
;
671 bool release
= false;
673 pr_debug("%s: state %d\n",
674 __func__
, csk
->com
.state
);
676 spin_lock_bh(&csk
->lock
);
677 switch (csk
->com
.state
) {
678 case CSK_STATE_ESTABLISHED
:
679 if (conn
->conn_state
== TARG_CONN_STATE_IN_LOGOUT
) {
680 csk
->com
.state
= CSK_STATE_CLOSING
;
681 cxgbit_send_halfclose(csk
);
683 csk
->com
.state
= CSK_STATE_ABORTING
;
684 cxgbit_send_abort_req(csk
);
687 case CSK_STATE_CLOSING
:
688 csk
->com
.state
= CSK_STATE_MORIBUND
;
689 cxgbit_send_halfclose(csk
);
695 pr_err("%s: csk %p; state %d\n",
696 __func__
, csk
, csk
->com
.state
);
698 spin_unlock_bh(&csk
->lock
);
704 static void cxgbit_set_emss(struct cxgbit_sock
*csk
, u16 opt
)
706 csk
->emss
= csk
->com
.cdev
->lldi
.mtus
[TCPOPT_MSS_G(opt
)] -
707 ((csk
->com
.remote_addr
.ss_family
== AF_INET
) ?
708 sizeof(struct iphdr
) : sizeof(struct ipv6hdr
)) -
709 sizeof(struct tcphdr
);
710 csk
->mss
= csk
->emss
;
711 if (TCPOPT_TSTAMP_G(opt
))
712 csk
->emss
-= round_up(TCPOLEN_TIMESTAMP
, 4);
716 pr_info("Warning: misaligned mtu idx %u mss %u emss=%u\n",
717 TCPOPT_MSS_G(opt
), csk
->mss
, csk
->emss
);
718 pr_debug("%s mss_idx %u mss %u emss=%u\n", __func__
, TCPOPT_MSS_G(opt
),
719 csk
->mss
, csk
->emss
);
722 static void cxgbit_free_skb(struct cxgbit_sock
*csk
)
726 __skb_queue_purge(&csk
->txq
);
727 __skb_queue_purge(&csk
->rxq
);
728 __skb_queue_purge(&csk
->backlogq
);
729 __skb_queue_purge(&csk
->ppodq
);
730 __skb_queue_purge(&csk
->skbq
);
732 while ((skb
= cxgbit_sock_dequeue_wr(csk
)))
735 __kfree_skb(csk
->lro_hskb
);
738 void _cxgbit_free_csk(struct kref
*kref
)
740 struct cxgbit_sock
*csk
;
741 struct cxgbit_device
*cdev
;
743 csk
= container_of(kref
, struct cxgbit_sock
, kref
);
745 pr_debug("%s csk %p state %d\n", __func__
, csk
, csk
->com
.state
);
747 if (csk
->com
.local_addr
.ss_family
== AF_INET6
) {
748 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
749 &csk
->com
.local_addr
;
750 cxgb4_clip_release(csk
->com
.cdev
->lldi
.ports
[0],
752 &sin6
->sin6_addr
.s6_addr
, 1);
755 cxgb4_remove_tid(csk
->com
.cdev
->lldi
.tids
, 0, csk
->tid
,
756 csk
->com
.local_addr
.ss_family
);
757 dst_release(csk
->dst
);
758 cxgb4_l2t_release(csk
->l2t
);
760 cdev
= csk
->com
.cdev
;
761 spin_lock_bh(&cdev
->cskq
.lock
);
762 list_del(&csk
->list
);
763 spin_unlock_bh(&cdev
->cskq
.lock
);
765 cxgbit_free_skb(csk
);
766 cxgbit_put_cdev(cdev
);
771 static void cxgbit_set_tcp_window(struct cxgbit_sock
*csk
, struct port_info
*pi
)
773 unsigned int linkspeed
;
776 linkspeed
= pi
->link_cfg
.speed
;
777 scale
= linkspeed
/ SPEED_10000
;
779 #define CXGBIT_10G_RCV_WIN (256 * 1024)
780 csk
->rcv_win
= CXGBIT_10G_RCV_WIN
;
782 csk
->rcv_win
*= scale
;
784 #define CXGBIT_10G_SND_WIN (256 * 1024)
785 csk
->snd_win
= CXGBIT_10G_SND_WIN
;
787 csk
->snd_win
*= scale
;
789 pr_debug("%s snd_win %d rcv_win %d\n",
790 __func__
, csk
->snd_win
, csk
->rcv_win
);
793 #ifdef CONFIG_CHELSIO_T4_DCB
794 static u8
cxgbit_get_iscsi_dcb_state(struct net_device
*ndev
)
796 return ndev
->dcbnl_ops
->getstate(ndev
);
799 static int cxgbit_select_priority(int pri_mask
)
804 return (ffs(pri_mask
) - 1);
807 static u8
cxgbit_get_iscsi_dcb_priority(struct net_device
*ndev
, u16 local_port
)
812 struct dcb_app iscsi_dcb_app
= {
813 .protocol
= local_port
816 ret
= (int)ndev
->dcbnl_ops
->getcap(ndev
, DCB_CAP_ATTR_DCBX
, &caps
);
821 if (caps
& DCB_CAP_DCBX_VER_IEEE
) {
822 iscsi_dcb_app
.selector
= IEEE_8021QAZ_APP_SEL_ANY
;
824 ret
= dcb_ieee_getapp_mask(ndev
, &iscsi_dcb_app
);
826 } else if (caps
& DCB_CAP_DCBX_VER_CEE
) {
827 iscsi_dcb_app
.selector
= DCB_APP_IDTYPE_PORTNUM
;
829 ret
= dcb_getapp(ndev
, &iscsi_dcb_app
);
832 pr_info("iSCSI priority is set to %u\n", cxgbit_select_priority(ret
));
834 return cxgbit_select_priority(ret
);
839 cxgbit_offload_init(struct cxgbit_sock
*csk
, int iptype
, __u8
*peer_ip
,
840 u16 local_port
, struct dst_entry
*dst
,
841 struct cxgbit_device
*cdev
)
845 struct net_device
*ndev
;
846 u16 rxq_idx
, port_id
;
847 #ifdef CONFIG_CHELSIO_T4_DCB
851 n
= dst_neigh_lookup(dst
, peer_ip
);
857 if (n
->dev
->flags
& IFF_LOOPBACK
) {
859 ndev
= cxgbit_ipv4_netdev(*(__be32
*)peer_ip
);
860 else if (IS_ENABLED(CONFIG_IPV6
))
861 ndev
= cxgbit_ipv6_netdev((struct in6_addr
*)peer_ip
);
870 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
,
874 csk
->mtu
= ndev
->mtu
;
875 csk
->tx_chan
= cxgb4_port_chan(ndev
);
876 csk
->smac_idx
= cxgb4_tp_smt_idx(cdev
->lldi
.adapter_type
,
877 cxgb4_port_viid(ndev
));
878 step
= cdev
->lldi
.ntxq
/
880 csk
->txq_idx
= cxgb4_port_idx(ndev
) * step
;
881 step
= cdev
->lldi
.nrxq
/
883 csk
->ctrlq_idx
= cxgb4_port_idx(ndev
);
884 csk
->rss_qid
= cdev
->lldi
.rxq_ids
[
885 cxgb4_port_idx(ndev
) * step
];
886 csk
->port_id
= cxgb4_port_idx(ndev
);
887 cxgbit_set_tcp_window(csk
,
888 (struct port_info
*)netdev_priv(ndev
));
890 ndev
= cxgbit_get_real_dev(n
->dev
);
896 #ifdef CONFIG_CHELSIO_T4_DCB
897 if (cxgbit_get_iscsi_dcb_state(ndev
))
898 priority
= cxgbit_get_iscsi_dcb_priority(ndev
,
901 csk
->dcb_priority
= priority
;
903 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
, n
, ndev
, priority
);
905 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
, n
, ndev
, 0);
909 port_id
= cxgb4_port_idx(ndev
);
910 csk
->mtu
= dst_mtu(dst
);
911 csk
->tx_chan
= cxgb4_port_chan(ndev
);
912 csk
->smac_idx
= cxgb4_tp_smt_idx(cdev
->lldi
.adapter_type
,
913 cxgb4_port_viid(ndev
));
914 step
= cdev
->lldi
.ntxq
/
916 csk
->txq_idx
= (port_id
* step
) +
917 (cdev
->selectq
[port_id
][0]++ % step
);
918 csk
->ctrlq_idx
= cxgb4_port_idx(ndev
);
919 step
= cdev
->lldi
.nrxq
/
921 rxq_idx
= (port_id
* step
) +
922 (cdev
->selectq
[port_id
][1]++ % step
);
923 csk
->rss_qid
= cdev
->lldi
.rxq_ids
[rxq_idx
];
924 csk
->port_id
= port_id
;
925 cxgbit_set_tcp_window(csk
,
926 (struct port_info
*)netdev_priv(ndev
));
935 int cxgbit_ofld_send(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
939 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
941 pr_err("%s - device not up - dropping\n", __func__
);
945 ret
= cxgb4_ofld_send(cdev
->lldi
.ports
[0], skb
);
948 return ret
< 0 ? ret
: 0;
951 static void cxgbit_release_tid(struct cxgbit_device
*cdev
, u32 tid
)
953 u32 len
= roundup(sizeof(struct cpl_tid_release
), 16);
956 skb
= alloc_skb(len
, GFP_ATOMIC
);
960 cxgb_mk_tid_release(skb
, len
, tid
, 0);
961 cxgbit_ofld_send(cdev
, skb
);
965 cxgbit_l2t_send(struct cxgbit_device
*cdev
, struct sk_buff
*skb
,
966 struct l2t_entry
*l2e
)
970 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
972 pr_err("%s - device not up - dropping\n", __func__
);
976 ret
= cxgb4_l2t_send(cdev
->lldi
.ports
[0], skb
, l2e
);
979 return ret
< 0 ? ret
: 0;
982 static void cxgbit_send_rx_credits(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
984 if (csk
->com
.state
!= CSK_STATE_ESTABLISHED
) {
989 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
993 * CPL connection rx data ack: host ->
994 * Send RX credits through an RX_DATA_ACK CPL message.
995 * Returns the number of credits sent.
997 int cxgbit_rx_data_ack(struct cxgbit_sock
*csk
)
1000 u32 len
= roundup(sizeof(struct cpl_rx_data_ack
), 16);
1003 skb
= alloc_skb(len
, GFP_KERNEL
);
1007 credit_dack
= RX_DACK_CHANGE_F
| RX_DACK_MODE_V(1) |
1008 RX_CREDITS_V(csk
->rx_credits
);
1010 cxgb_mk_rx_data_ack(skb
, len
, csk
->tid
, csk
->ctrlq_idx
,
1013 csk
->rx_credits
= 0;
1015 spin_lock_bh(&csk
->lock
);
1016 if (csk
->lock_owner
) {
1017 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_send_rx_credits
;
1018 __skb_queue_tail(&csk
->backlogq
, skb
);
1019 spin_unlock_bh(&csk
->lock
);
1023 cxgbit_send_rx_credits(csk
, skb
);
1024 spin_unlock_bh(&csk
->lock
);
1029 #define FLOWC_WR_NPARAMS_MIN 9
1030 #define FLOWC_WR_NPARAMS_MAX 11
1031 static int cxgbit_alloc_csk_skb(struct cxgbit_sock
*csk
)
1033 struct sk_buff
*skb
;
1037 flowclen
= offsetof(struct fw_flowc_wr
,
1038 mnemval
[FLOWC_WR_NPARAMS_MAX
]);
1040 len
= max_t(u32
, sizeof(struct cpl_abort_req
),
1041 sizeof(struct cpl_abort_rpl
));
1043 len
= max(len
, flowclen
);
1044 len
= roundup(len
, 16);
1046 for (i
= 0; i
< 3; i
++) {
1047 skb
= alloc_skb(len
, GFP_ATOMIC
);
1050 __skb_queue_tail(&csk
->skbq
, skb
);
1053 skb
= alloc_skb(LRO_SKB_MIN_HEADROOM
, GFP_ATOMIC
);
1057 memset(skb
->data
, 0, LRO_SKB_MIN_HEADROOM
);
1058 csk
->lro_hskb
= skb
;
1062 __skb_queue_purge(&csk
->skbq
);
1067 cxgbit_pass_accept_rpl(struct cxgbit_sock
*csk
, struct cpl_pass_accept_req
*req
)
1069 struct sk_buff
*skb
;
1070 const struct tcphdr
*tcph
;
1071 struct cpl_t5_pass_accept_rpl
*rpl5
;
1072 struct cxgb4_lld_info
*lldi
= &csk
->com
.cdev
->lldi
;
1073 unsigned int len
= roundup(sizeof(*rpl5
), 16);
1074 unsigned int mtu_idx
;
1080 pr_debug("%s csk %p tid %u\n", __func__
, csk
, csk
->tid
);
1082 skb
= alloc_skb(len
, GFP_ATOMIC
);
1084 cxgbit_put_csk(csk
);
1088 rpl5
= __skb_put(skb
, len
);
1089 memset(rpl5
, 0, len
);
1091 INIT_TP_WR(rpl5
, csk
->tid
);
1092 OPCODE_TID(rpl5
) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL
,
1094 cxgb_best_mtu(csk
->com
.cdev
->lldi
.mtus
, csk
->mtu
, &mtu_idx
,
1096 (csk
->com
.remote_addr
.ss_family
== AF_INET
) ? 0 : 1);
1097 wscale
= cxgb_compute_wscale(csk
->rcv_win
);
1099 * Specify the largest window that will fit in opt0. The
1100 * remainder will be specified in the rx_data_ack.
1102 win
= csk
->rcv_win
>> 10;
1103 if (win
> RCV_BUFSIZ_M
)
1105 opt0
= TCAM_BYPASS_F
|
1106 WND_SCALE_V(wscale
) |
1107 MSS_IDX_V(mtu_idx
) |
1108 L2T_IDX_V(csk
->l2t
->idx
) |
1109 TX_CHAN_V(csk
->tx_chan
) |
1110 SMAC_SEL_V(csk
->smac_idx
) |
1111 DSCP_V(csk
->tos
>> 2) |
1112 ULP_MODE_V(ULP_MODE_ISCSI
) |
1115 opt2
= RX_CHANNEL_V(0) |
1116 RSS_QUEUE_VALID_F
| RSS_QUEUE_V(csk
->rss_qid
);
1118 if (!is_t5(lldi
->adapter_type
))
1119 opt2
|= RX_FC_DISABLE_F
;
1121 if (req
->tcpopt
.tstamp
)
1122 opt2
|= TSTAMPS_EN_F
;
1123 if (req
->tcpopt
.sack
)
1126 opt2
|= WND_SCALE_EN_F
;
1128 hlen
= ntohl(req
->hdr_len
);
1130 if (is_t5(lldi
->adapter_type
))
1131 tcph
= (struct tcphdr
*)((u8
*)(req
+ 1) +
1132 ETH_HDR_LEN_G(hlen
) + IP_HDR_LEN_G(hlen
));
1134 tcph
= (struct tcphdr
*)((u8
*)(req
+ 1) +
1135 T6_ETH_HDR_LEN_G(hlen
) + T6_IP_HDR_LEN_G(hlen
));
1137 if (tcph
->ece
&& tcph
->cwr
)
1138 opt2
|= CCTRL_ECN_V(1);
1140 opt2
|= RX_COALESCE_V(3);
1141 opt2
|= CONG_CNTRL_V(CONG_ALG_NEWRENO
);
1144 rpl5
->iss
= cpu_to_be32((prandom_u32() & ~7UL) - 1);
1146 opt2
|= T5_OPT_2_VALID_F
;
1148 rpl5
->opt0
= cpu_to_be64(opt0
);
1149 rpl5
->opt2
= cpu_to_be32(opt2
);
1150 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, csk
->ctrlq_idx
);
1151 t4_set_arp_err_handler(skb
, NULL
, cxgbit_arp_failure_discard
);
1152 cxgbit_l2t_send(csk
->com
.cdev
, skb
, csk
->l2t
);
1156 cxgbit_pass_accept_req(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1158 struct cxgbit_sock
*csk
= NULL
;
1159 struct cxgbit_np
*cnp
;
1160 struct cpl_pass_accept_req
*req
= cplhdr(skb
);
1161 unsigned int stid
= PASS_OPEN_TID_G(ntohl(req
->tos_stid
));
1162 struct tid_info
*t
= cdev
->lldi
.tids
;
1163 unsigned int tid
= GET_TID(req
);
1164 u16 peer_mss
= ntohs(req
->tcpopt
.mss
);
1165 unsigned short hdrs
;
1167 struct dst_entry
*dst
;
1168 __u8 local_ip
[16], peer_ip
[16];
1169 __be16 local_port
, peer_port
;
1173 pr_debug("%s: cdev = %p; stid = %u; tid = %u\n",
1174 __func__
, cdev
, stid
, tid
);
1176 cnp
= lookup_stid(t
, stid
);
1178 pr_err("%s connect request on invalid stid %d\n",
1183 if (cnp
->com
.state
!= CSK_STATE_LISTEN
) {
1184 pr_err("%s - listening parent not in CSK_STATE_LISTEN\n",
1189 csk
= lookup_tid(t
, tid
);
1191 pr_err("%s csk not null tid %u\n",
1196 cxgb_get_4tuple(req
, cdev
->lldi
.adapter_type
, &iptype
, local_ip
,
1197 peer_ip
, &local_port
, &peer_port
);
1199 /* Find output route */
1201 pr_debug("%s parent sock %p tid %u laddr %pI4 raddr %pI4 "
1202 "lport %d rport %d peer_mss %d\n"
1203 , __func__
, cnp
, tid
,
1204 local_ip
, peer_ip
, ntohs(local_port
),
1205 ntohs(peer_port
), peer_mss
);
1206 dst
= cxgb_find_route(&cdev
->lldi
, cxgbit_get_real_dev
,
1207 *(__be32
*)local_ip
,
1209 local_port
, peer_port
,
1210 PASS_OPEN_TOS_G(ntohl(req
->tos_stid
)));
1212 pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 "
1213 "lport %d rport %d peer_mss %d\n"
1214 , __func__
, cnp
, tid
,
1215 local_ip
, peer_ip
, ntohs(local_port
),
1216 ntohs(peer_port
), peer_mss
);
1217 dst
= cxgb_find_route6(&cdev
->lldi
, cxgbit_get_real_dev
,
1219 local_port
, peer_port
,
1220 PASS_OPEN_TOS_G(ntohl(req
->tos_stid
)),
1221 ((struct sockaddr_in6
*)
1222 &cnp
->com
.local_addr
)->sin6_scope_id
);
1225 pr_err("%s - failed to find dst entry!\n",
1230 csk
= kzalloc(sizeof(*csk
), GFP_ATOMIC
);
1236 ret
= cxgbit_offload_init(csk
, iptype
, peer_ip
, ntohs(local_port
),
1239 pr_err("%s - failed to allocate l2t entry!\n",
1246 kref_init(&csk
->kref
);
1247 init_completion(&csk
->com
.wr_wait
.completion
);
1249 INIT_LIST_HEAD(&csk
->accept_node
);
1251 hdrs
= (iptype
== 4 ? sizeof(struct iphdr
) : sizeof(struct ipv6hdr
)) +
1252 sizeof(struct tcphdr
) + (req
->tcpopt
.tstamp
? 12 : 0);
1253 if (peer_mss
&& csk
->mtu
> (peer_mss
+ hdrs
))
1254 csk
->mtu
= peer_mss
+ hdrs
;
1256 csk
->com
.state
= CSK_STATE_CONNECTING
;
1257 csk
->com
.cdev
= cdev
;
1259 csk
->tos
= PASS_OPEN_TOS_G(ntohl(req
->tos_stid
));
1262 csk
->wr_cred
= cdev
->lldi
.wr_cred
-
1263 DIV_ROUND_UP(sizeof(struct cpl_abort_req
), 16);
1264 csk
->wr_max_cred
= csk
->wr_cred
;
1265 csk
->wr_una_cred
= 0;
1268 struct sockaddr_in
*sin
= (struct sockaddr_in
*)
1269 &csk
->com
.local_addr
;
1270 sin
->sin_family
= AF_INET
;
1271 sin
->sin_port
= local_port
;
1272 sin
->sin_addr
.s_addr
= *(__be32
*)local_ip
;
1274 sin
= (struct sockaddr_in
*)&csk
->com
.remote_addr
;
1275 sin
->sin_family
= AF_INET
;
1276 sin
->sin_port
= peer_port
;
1277 sin
->sin_addr
.s_addr
= *(__be32
*)peer_ip
;
1279 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
1280 &csk
->com
.local_addr
;
1282 sin6
->sin6_family
= PF_INET6
;
1283 sin6
->sin6_port
= local_port
;
1284 memcpy(sin6
->sin6_addr
.s6_addr
, local_ip
, 16);
1285 cxgb4_clip_get(cdev
->lldi
.ports
[0],
1286 (const u32
*)&sin6
->sin6_addr
.s6_addr
,
1289 sin6
= (struct sockaddr_in6
*)&csk
->com
.remote_addr
;
1290 sin6
->sin6_family
= PF_INET6
;
1291 sin6
->sin6_port
= peer_port
;
1292 memcpy(sin6
->sin6_addr
.s6_addr
, peer_ip
, 16);
1295 skb_queue_head_init(&csk
->rxq
);
1296 skb_queue_head_init(&csk
->txq
);
1297 skb_queue_head_init(&csk
->ppodq
);
1298 skb_queue_head_init(&csk
->backlogq
);
1299 skb_queue_head_init(&csk
->skbq
);
1300 cxgbit_sock_reset_wr_list(csk
);
1301 spin_lock_init(&csk
->lock
);
1302 init_waitqueue_head(&csk
->waitq
);
1303 init_waitqueue_head(&csk
->ack_waitq
);
1304 csk
->lock_owner
= false;
1306 if (cxgbit_alloc_csk_skb(csk
)) {
1312 cxgbit_get_cdev(cdev
);
1314 spin_lock(&cdev
->cskq
.lock
);
1315 list_add_tail(&csk
->list
, &cdev
->cskq
.list
);
1316 spin_unlock(&cdev
->cskq
.lock
);
1317 cxgb4_insert_tid(t
, csk
, tid
, csk
->com
.local_addr
.ss_family
);
1318 cxgbit_pass_accept_rpl(csk
, req
);
1322 cxgbit_release_tid(cdev
, tid
);
1328 cxgbit_tx_flowc_wr_credits(struct cxgbit_sock
*csk
, u32
*nparamsp
,
1331 u32 nparams
, flowclen16
, flowclen
;
1333 nparams
= FLOWC_WR_NPARAMS_MIN
;
1335 if (csk
->snd_wscale
)
1338 #ifdef CONFIG_CHELSIO_T4_DCB
1341 flowclen
= offsetof(struct fw_flowc_wr
, mnemval
[nparams
]);
1342 flowclen16
= DIV_ROUND_UP(flowclen
, 16);
1343 flowclen
= flowclen16
* 16;
1345 * Return the number of 16-byte credits used by the flowc request.
1346 * Pass back the nparams and actual flowc length if requested.
1349 *nparamsp
= nparams
;
1351 *flowclenp
= flowclen
;
1355 u32
cxgbit_send_tx_flowc_wr(struct cxgbit_sock
*csk
)
1357 struct cxgbit_device
*cdev
= csk
->com
.cdev
;
1358 struct fw_flowc_wr
*flowc
;
1359 u32 nparams
, flowclen16
, flowclen
;
1360 struct sk_buff
*skb
;
1363 #ifdef CONFIG_CHELSIO_T4_DCB
1364 u16 vlan
= ((struct l2t_entry
*)csk
->l2t
)->vlan
;
1367 flowclen16
= cxgbit_tx_flowc_wr_credits(csk
, &nparams
, &flowclen
);
1369 skb
= __skb_dequeue(&csk
->skbq
);
1370 flowc
= __skb_put(skb
, flowclen
);
1371 memset(flowc
, 0, flowclen
);
1373 flowc
->op_to_nparams
= cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR
) |
1374 FW_FLOWC_WR_NPARAMS_V(nparams
));
1375 flowc
->flowid_len16
= cpu_to_be32(FW_WR_LEN16_V(flowclen16
) |
1376 FW_WR_FLOWID_V(csk
->tid
));
1377 flowc
->mnemval
[0].mnemonic
= FW_FLOWC_MNEM_PFNVFN
;
1378 flowc
->mnemval
[0].val
= cpu_to_be32(FW_PFVF_CMD_PFN_V
1379 (csk
->com
.cdev
->lldi
.pf
));
1380 flowc
->mnemval
[1].mnemonic
= FW_FLOWC_MNEM_CH
;
1381 flowc
->mnemval
[1].val
= cpu_to_be32(csk
->tx_chan
);
1382 flowc
->mnemval
[2].mnemonic
= FW_FLOWC_MNEM_PORT
;
1383 flowc
->mnemval
[2].val
= cpu_to_be32(csk
->tx_chan
);
1384 flowc
->mnemval
[3].mnemonic
= FW_FLOWC_MNEM_IQID
;
1385 flowc
->mnemval
[3].val
= cpu_to_be32(csk
->rss_qid
);
1386 flowc
->mnemval
[4].mnemonic
= FW_FLOWC_MNEM_SNDNXT
;
1387 flowc
->mnemval
[4].val
= cpu_to_be32(csk
->snd_nxt
);
1388 flowc
->mnemval
[5].mnemonic
= FW_FLOWC_MNEM_RCVNXT
;
1389 flowc
->mnemval
[5].val
= cpu_to_be32(csk
->rcv_nxt
);
1390 flowc
->mnemval
[6].mnemonic
= FW_FLOWC_MNEM_SNDBUF
;
1391 flowc
->mnemval
[6].val
= cpu_to_be32(csk
->snd_win
);
1392 flowc
->mnemval
[7].mnemonic
= FW_FLOWC_MNEM_MSS
;
1393 flowc
->mnemval
[7].val
= cpu_to_be32(csk
->emss
);
1395 flowc
->mnemval
[8].mnemonic
= FW_FLOWC_MNEM_TXDATAPLEN_MAX
;
1396 if (test_bit(CDEV_ISO_ENABLE
, &cdev
->flags
))
1397 flowc
->mnemval
[8].val
= cpu_to_be32(CXGBIT_MAX_ISO_PAYLOAD
);
1399 flowc
->mnemval
[8].val
= cpu_to_be32(16384);
1403 if (csk
->snd_wscale
) {
1404 flowc
->mnemval
[index
].mnemonic
= FW_FLOWC_MNEM_RCV_SCALE
;
1405 flowc
->mnemval
[index
].val
= cpu_to_be32(csk
->snd_wscale
);
1409 #ifdef CONFIG_CHELSIO_T4_DCB
1410 flowc
->mnemval
[index
].mnemonic
= FW_FLOWC_MNEM_DCBPRIO
;
1411 if (vlan
== VLAN_NONE
) {
1412 pr_warn("csk %u without VLAN Tag on DCB Link\n", csk
->tid
);
1413 flowc
->mnemval
[index
].val
= cpu_to_be32(0);
1415 flowc
->mnemval
[index
].val
= cpu_to_be32(
1416 (vlan
& VLAN_PRIO_MASK
) >> VLAN_PRIO_SHIFT
);
1419 pr_debug("%s: csk %p; tx_chan = %u; rss_qid = %u; snd_seq = %u;"
1420 " rcv_seq = %u; snd_win = %u; emss = %u\n",
1421 __func__
, csk
, csk
->tx_chan
, csk
->rss_qid
, csk
->snd_nxt
,
1422 csk
->rcv_nxt
, csk
->snd_win
, csk
->emss
);
1423 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->txq_idx
);
1424 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1428 int cxgbit_setup_conn_digest(struct cxgbit_sock
*csk
)
1430 struct sk_buff
*skb
;
1431 struct cpl_set_tcb_field
*req
;
1432 u8 hcrc
= csk
->submode
& CXGBIT_SUBMODE_HCRC
;
1433 u8 dcrc
= csk
->submode
& CXGBIT_SUBMODE_DCRC
;
1434 unsigned int len
= roundup(sizeof(*req
), 16);
1437 skb
= alloc_skb(len
, GFP_KERNEL
);
1441 /* set up ulp submode */
1442 req
= __skb_put(skb
, len
);
1443 memset(req
, 0, len
);
1445 INIT_TP_WR(req
, csk
->tid
);
1446 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, csk
->tid
));
1447 req
->reply_ctrl
= htons(NO_REPLY_V(0) | QUEUENO_V(csk
->rss_qid
));
1448 req
->word_cookie
= htons(0);
1449 req
->mask
= cpu_to_be64(0x3 << 4);
1450 req
->val
= cpu_to_be64(((hcrc
? ULP_CRC_HEADER
: 0) |
1451 (dcrc
? ULP_CRC_DATA
: 0)) << 4);
1452 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->ctrlq_idx
);
1454 cxgbit_get_csk(csk
);
1455 cxgbit_init_wr_wait(&csk
->com
.wr_wait
);
1457 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1459 ret
= cxgbit_wait_for_reply(csk
->com
.cdev
,
1461 csk
->tid
, 5, __func__
);
1468 int cxgbit_setup_conn_pgidx(struct cxgbit_sock
*csk
, u32 pg_idx
)
1470 struct sk_buff
*skb
;
1471 struct cpl_set_tcb_field
*req
;
1472 unsigned int len
= roundup(sizeof(*req
), 16);
1475 skb
= alloc_skb(len
, GFP_KERNEL
);
1479 req
= __skb_put(skb
, len
);
1480 memset(req
, 0, len
);
1482 INIT_TP_WR(req
, csk
->tid
);
1483 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, csk
->tid
));
1484 req
->reply_ctrl
= htons(NO_REPLY_V(0) | QUEUENO_V(csk
->rss_qid
));
1485 req
->word_cookie
= htons(0);
1486 req
->mask
= cpu_to_be64(0x3 << 8);
1487 req
->val
= cpu_to_be64(pg_idx
<< 8);
1488 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->ctrlq_idx
);
1490 cxgbit_get_csk(csk
);
1491 cxgbit_init_wr_wait(&csk
->com
.wr_wait
);
1493 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1495 ret
= cxgbit_wait_for_reply(csk
->com
.cdev
,
1497 csk
->tid
, 5, __func__
);
1505 cxgbit_pass_open_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1507 struct cpl_pass_open_rpl
*rpl
= cplhdr(skb
);
1508 struct tid_info
*t
= cdev
->lldi
.tids
;
1509 unsigned int stid
= GET_TID(rpl
);
1510 struct cxgbit_np
*cnp
= lookup_stid(t
, stid
);
1512 pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1513 __func__
, cnp
, stid
, rpl
->status
);
1516 pr_info("%s stid %d lookup failure\n", __func__
, stid
);
1520 cxgbit_wake_up(&cnp
->com
.wr_wait
, __func__
, rpl
->status
);
1521 cxgbit_put_cnp(cnp
);
1525 cxgbit_close_listsrv_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1527 struct cpl_close_listsvr_rpl
*rpl
= cplhdr(skb
);
1528 struct tid_info
*t
= cdev
->lldi
.tids
;
1529 unsigned int stid
= GET_TID(rpl
);
1530 struct cxgbit_np
*cnp
= lookup_stid(t
, stid
);
1532 pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1533 __func__
, cnp
, stid
, rpl
->status
);
1536 pr_info("%s stid %d lookup failure\n", __func__
, stid
);
1540 cxgbit_wake_up(&cnp
->com
.wr_wait
, __func__
, rpl
->status
);
1541 cxgbit_put_cnp(cnp
);
1545 cxgbit_pass_establish(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1547 struct cpl_pass_establish
*req
= cplhdr(skb
);
1548 struct tid_info
*t
= cdev
->lldi
.tids
;
1549 unsigned int tid
= GET_TID(req
);
1550 struct cxgbit_sock
*csk
;
1551 struct cxgbit_np
*cnp
;
1552 u16 tcp_opt
= be16_to_cpu(req
->tcp_opt
);
1553 u32 snd_isn
= be32_to_cpu(req
->snd_isn
);
1554 u32 rcv_isn
= be32_to_cpu(req
->rcv_isn
);
1556 csk
= lookup_tid(t
, tid
);
1557 if (unlikely(!csk
)) {
1558 pr_err("can't find connection for tid %u.\n", tid
);
1563 pr_debug("%s: csk %p; tid %u; cnp %p\n",
1564 __func__
, csk
, tid
, cnp
);
1566 csk
->write_seq
= snd_isn
;
1567 csk
->snd_una
= snd_isn
;
1568 csk
->snd_nxt
= snd_isn
;
1570 csk
->rcv_nxt
= rcv_isn
;
1572 if (csk
->rcv_win
> (RCV_BUFSIZ_M
<< 10))
1573 csk
->rx_credits
= (csk
->rcv_win
- (RCV_BUFSIZ_M
<< 10));
1575 csk
->snd_wscale
= TCPOPT_SND_WSCALE_G(tcp_opt
);
1576 cxgbit_set_emss(csk
, tcp_opt
);
1577 dst_confirm(csk
->dst
);
1578 csk
->com
.state
= CSK_STATE_ESTABLISHED
;
1579 spin_lock_bh(&cnp
->np_accept_lock
);
1580 list_add_tail(&csk
->accept_node
, &cnp
->np_accept_list
);
1581 spin_unlock_bh(&cnp
->np_accept_lock
);
1582 complete(&cnp
->accept_comp
);
1587 static void cxgbit_queue_rx_skb(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1589 cxgbit_skcb_flags(skb
) = 0;
1590 spin_lock_bh(&csk
->rxq
.lock
);
1591 __skb_queue_tail(&csk
->rxq
, skb
);
1592 spin_unlock_bh(&csk
->rxq
.lock
);
1593 wake_up(&csk
->waitq
);
1596 static void cxgbit_peer_close(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1598 pr_debug("%s: csk %p; tid %u; state %d\n",
1599 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1601 switch (csk
->com
.state
) {
1602 case CSK_STATE_ESTABLISHED
:
1603 csk
->com
.state
= CSK_STATE_CLOSING
;
1604 cxgbit_queue_rx_skb(csk
, skb
);
1606 case CSK_STATE_CLOSING
:
1607 /* simultaneous close */
1608 csk
->com
.state
= CSK_STATE_MORIBUND
;
1610 case CSK_STATE_MORIBUND
:
1611 csk
->com
.state
= CSK_STATE_DEAD
;
1612 cxgbit_put_csk(csk
);
1614 case CSK_STATE_ABORTING
:
1617 pr_info("%s: cpl_peer_close in bad state %d\n",
1618 __func__
, csk
->com
.state
);
1624 static void cxgbit_close_con_rpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1626 pr_debug("%s: csk %p; tid %u; state %d\n",
1627 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1629 switch (csk
->com
.state
) {
1630 case CSK_STATE_CLOSING
:
1631 csk
->com
.state
= CSK_STATE_MORIBUND
;
1633 case CSK_STATE_MORIBUND
:
1634 csk
->com
.state
= CSK_STATE_DEAD
;
1635 cxgbit_put_csk(csk
);
1637 case CSK_STATE_ABORTING
:
1638 case CSK_STATE_DEAD
:
1641 pr_info("%s: cpl_close_con_rpl in bad state %d\n",
1642 __func__
, csk
->com
.state
);
1648 static void cxgbit_abort_req_rss(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1650 struct cpl_abort_req_rss
*hdr
= cplhdr(skb
);
1651 unsigned int tid
= GET_TID(hdr
);
1652 struct sk_buff
*rpl_skb
;
1653 bool release
= false;
1654 bool wakeup_thread
= false;
1655 u32 len
= roundup(sizeof(struct cpl_abort_rpl
), 16);
1657 pr_debug("%s: csk %p; tid %u; state %d\n",
1658 __func__
, csk
, tid
, csk
->com
.state
);
1660 if (cxgb_is_neg_adv(hdr
->status
)) {
1661 pr_err("%s: got neg advise %d on tid %u\n",
1662 __func__
, hdr
->status
, tid
);
1666 switch (csk
->com
.state
) {
1667 case CSK_STATE_CONNECTING
:
1668 case CSK_STATE_MORIBUND
:
1669 csk
->com
.state
= CSK_STATE_DEAD
;
1672 case CSK_STATE_ESTABLISHED
:
1673 csk
->com
.state
= CSK_STATE_DEAD
;
1674 wakeup_thread
= true;
1676 case CSK_STATE_CLOSING
:
1677 csk
->com
.state
= CSK_STATE_DEAD
;
1681 case CSK_STATE_ABORTING
:
1684 pr_info("%s: cpl_abort_req_rss in bad state %d\n",
1685 __func__
, csk
->com
.state
);
1686 csk
->com
.state
= CSK_STATE_DEAD
;
1689 __skb_queue_purge(&csk
->txq
);
1691 if (!test_and_set_bit(CSK_TX_DATA_SENT
, &csk
->com
.flags
))
1692 cxgbit_send_tx_flowc_wr(csk
);
1694 rpl_skb
= __skb_dequeue(&csk
->skbq
);
1696 cxgb_mk_abort_rpl(rpl_skb
, len
, csk
->tid
, csk
->txq_idx
);
1697 cxgbit_ofld_send(csk
->com
.cdev
, rpl_skb
);
1699 if (wakeup_thread
) {
1700 cxgbit_queue_rx_skb(csk
, skb
);
1705 cxgbit_put_csk(csk
);
1710 static void cxgbit_abort_rpl_rss(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1712 pr_debug("%s: csk %p; tid %u; state %d\n",
1713 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1715 switch (csk
->com
.state
) {
1716 case CSK_STATE_ABORTING
:
1717 csk
->com
.state
= CSK_STATE_DEAD
;
1718 cxgbit_put_csk(csk
);
1721 pr_info("%s: cpl_abort_rpl_rss in state %d\n",
1722 __func__
, csk
->com
.state
);
1728 static bool cxgbit_credit_err(const struct cxgbit_sock
*csk
)
1730 const struct sk_buff
*skb
= csk
->wr_pending_head
;
1733 if (unlikely(csk
->wr_cred
> csk
->wr_max_cred
)) {
1734 pr_err("csk 0x%p, tid %u, credit %u > %u\n",
1735 csk
, csk
->tid
, csk
->wr_cred
, csk
->wr_max_cred
);
1740 credit
+= (__force u32
)skb
->csum
;
1741 skb
= cxgbit_skcb_tx_wr_next(skb
);
1744 if (unlikely((csk
->wr_cred
+ credit
) != csk
->wr_max_cred
)) {
1745 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1746 csk
, csk
->tid
, csk
->wr_cred
,
1747 credit
, csk
->wr_max_cred
);
1755 static void cxgbit_fw4_ack(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1757 struct cpl_fw4_ack
*rpl
= (struct cpl_fw4_ack
*)cplhdr(skb
);
1758 u32 credits
= rpl
->credits
;
1759 u32 snd_una
= ntohl(rpl
->snd_una
);
1761 csk
->wr_cred
+= credits
;
1762 if (csk
->wr_una_cred
> (csk
->wr_max_cred
- csk
->wr_cred
))
1763 csk
->wr_una_cred
= csk
->wr_max_cred
- csk
->wr_cred
;
1766 struct sk_buff
*p
= cxgbit_sock_peek_wr(csk
);
1767 const u32 csum
= (__force u32
)p
->csum
;
1770 pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n",
1771 csk
, csk
->tid
, credits
,
1772 csk
->wr_cred
, csk
->wr_una_cred
);
1776 if (unlikely(credits
< csum
)) {
1777 pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n",
1779 credits
, csk
->wr_cred
, csk
->wr_una_cred
,
1781 p
->csum
= (__force __wsum
)(csum
- credits
);
1785 cxgbit_sock_dequeue_wr(csk
);
1790 if (unlikely(cxgbit_credit_err(csk
))) {
1791 cxgbit_queue_rx_skb(csk
, skb
);
1795 if (rpl
->seq_vld
& CPL_FW4_ACK_FLAGS_SEQVAL
) {
1796 if (unlikely(before(snd_una
, csk
->snd_una
))) {
1797 pr_warn("csk 0x%p,%u, snd_una %u/%u.",
1798 csk
, csk
->tid
, snd_una
,
1803 if (csk
->snd_una
!= snd_una
) {
1804 csk
->snd_una
= snd_una
;
1805 dst_confirm(csk
->dst
);
1806 wake_up(&csk
->ack_waitq
);
1810 if (skb_queue_len(&csk
->txq
))
1811 cxgbit_push_tx_frames(csk
);
1817 static void cxgbit_set_tcb_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1819 struct cxgbit_sock
*csk
;
1820 struct cpl_set_tcb_rpl
*rpl
= (struct cpl_set_tcb_rpl
*)skb
->data
;
1821 unsigned int tid
= GET_TID(rpl
);
1822 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1823 struct tid_info
*t
= lldi
->tids
;
1825 csk
= lookup_tid(t
, tid
);
1827 pr_err("can't find connection for tid %u.\n", tid
);
1829 cxgbit_wake_up(&csk
->com
.wr_wait
, __func__
, rpl
->status
);
1831 cxgbit_put_csk(csk
);
1834 static void cxgbit_rx_data(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1836 struct cxgbit_sock
*csk
;
1837 struct cpl_rx_data
*cpl
= cplhdr(skb
);
1838 unsigned int tid
= GET_TID(cpl
);
1839 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1840 struct tid_info
*t
= lldi
->tids
;
1842 csk
= lookup_tid(t
, tid
);
1843 if (unlikely(!csk
)) {
1844 pr_err("can't find conn. for tid %u.\n", tid
);
1848 cxgbit_queue_rx_skb(csk
, skb
);
1855 __cxgbit_process_rx_cpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1857 spin_lock(&csk
->lock
);
1858 if (csk
->lock_owner
) {
1859 __skb_queue_tail(&csk
->backlogq
, skb
);
1860 spin_unlock(&csk
->lock
);
1864 cxgbit_skcb_rx_backlog_fn(skb
)(csk
, skb
);
1865 spin_unlock(&csk
->lock
);
1868 static void cxgbit_process_rx_cpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1870 cxgbit_get_csk(csk
);
1871 __cxgbit_process_rx_cpl(csk
, skb
);
1872 cxgbit_put_csk(csk
);
1875 static void cxgbit_rx_cpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1877 struct cxgbit_sock
*csk
;
1878 struct cpl_tx_data
*cpl
= cplhdr(skb
);
1879 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1880 struct tid_info
*t
= lldi
->tids
;
1881 unsigned int tid
= GET_TID(cpl
);
1882 u8 opcode
= cxgbit_skcb_rx_opcode(skb
);
1887 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_fw4_ack
;
1890 case CPL_PEER_CLOSE
:
1891 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_peer_close
;
1893 case CPL_CLOSE_CON_RPL
:
1894 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_close_con_rpl
;
1896 case CPL_ABORT_REQ_RSS
:
1897 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_abort_req_rss
;
1899 case CPL_ABORT_RPL_RSS
:
1900 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_abort_rpl_rss
;
1906 csk
= lookup_tid(t
, tid
);
1907 if (unlikely(!csk
)) {
1908 pr_err("can't find conn. for tid %u.\n", tid
);
1913 cxgbit_process_rx_cpl(csk
, skb
);
1915 __cxgbit_process_rx_cpl(csk
, skb
);
1922 cxgbit_cplhandler_func cxgbit_cplhandlers
[NUM_CPL_CMDS
] = {
1923 [CPL_PASS_OPEN_RPL
] = cxgbit_pass_open_rpl
,
1924 [CPL_CLOSE_LISTSRV_RPL
] = cxgbit_close_listsrv_rpl
,
1925 [CPL_PASS_ACCEPT_REQ
] = cxgbit_pass_accept_req
,
1926 [CPL_PASS_ESTABLISH
] = cxgbit_pass_establish
,
1927 [CPL_SET_TCB_RPL
] = cxgbit_set_tcb_rpl
,
1928 [CPL_RX_DATA
] = cxgbit_rx_data
,
1929 [CPL_FW4_ACK
] = cxgbit_rx_cpl
,
1930 [CPL_PEER_CLOSE
] = cxgbit_rx_cpl
,
1931 [CPL_CLOSE_CON_RPL
] = cxgbit_rx_cpl
,
1932 [CPL_ABORT_REQ_RSS
] = cxgbit_rx_cpl
,
1933 [CPL_ABORT_RPL_RSS
] = cxgbit_rx_cpl
,