2 * Copyright (c) 2016 Chelsio Communications, Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 #include <linux/module.h>
10 #include <linux/list.h>
11 #include <linux/workqueue.h>
12 #include <linux/skbuff.h>
13 #include <linux/timer.h>
14 #include <linux/notifier.h>
15 #include <linux/inetdevice.h>
17 #include <linux/tcp.h>
18 #include <linux/if_vlan.h>
20 #include <net/neighbour.h>
21 #include <net/netevent.h>
22 #include <net/route.h>
24 #include <net/ip6_route.h>
25 #include <net/addrconf.h>
27 #include <libcxgb_cm.h>
31 static void cxgbit_init_wr_wait(struct cxgbit_wr_wait
*wr_waitp
)
34 reinit_completion(&wr_waitp
->completion
);
38 cxgbit_wake_up(struct cxgbit_wr_wait
*wr_waitp
, const char *func
, u8 ret
)
40 if (ret
== CPL_ERR_NONE
)
46 pr_err("%s: err:%u", func
, ret
);
48 complete(&wr_waitp
->completion
);
52 cxgbit_wait_for_reply(struct cxgbit_device
*cdev
,
53 struct cxgbit_wr_wait
*wr_waitp
, u32 tid
, u32 timeout
,
58 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
63 ret
= wait_for_completion_timeout(&wr_waitp
->completion
, timeout
* HZ
);
65 pr_info("%s - Device %s not responding tid %u\n",
66 func
, pci_name(cdev
->lldi
.pdev
), tid
);
67 wr_waitp
->ret
= -ETIMEDOUT
;
71 pr_info("%s: FW reply %d tid %u\n",
72 pci_name(cdev
->lldi
.pdev
), wr_waitp
->ret
, tid
);
76 static int cxgbit_np_hashfn(const struct cxgbit_np
*cnp
)
78 return ((unsigned long)cnp
>> 10) & (NP_INFO_HASH_SIZE
- 1);
81 static struct np_info
*
82 cxgbit_np_hash_add(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
,
85 struct np_info
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
88 int bucket
= cxgbit_np_hashfn(cnp
);
92 spin_lock(&cdev
->np_lock
);
93 p
->next
= cdev
->np_hash_tab
[bucket
];
94 cdev
->np_hash_tab
[bucket
] = p
;
95 spin_unlock(&cdev
->np_lock
);
102 cxgbit_np_hash_find(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
104 int stid
= -1, bucket
= cxgbit_np_hashfn(cnp
);
107 spin_lock(&cdev
->np_lock
);
108 for (p
= cdev
->np_hash_tab
[bucket
]; p
; p
= p
->next
) {
114 spin_unlock(&cdev
->np_lock
);
119 static int cxgbit_np_hash_del(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
121 int stid
= -1, bucket
= cxgbit_np_hashfn(cnp
);
122 struct np_info
*p
, **prev
= &cdev
->np_hash_tab
[bucket
];
124 spin_lock(&cdev
->np_lock
);
125 for (p
= *prev
; p
; prev
= &p
->next
, p
= p
->next
) {
133 spin_unlock(&cdev
->np_lock
);
138 void _cxgbit_free_cnp(struct kref
*kref
)
140 struct cxgbit_np
*cnp
;
142 cnp
= container_of(kref
, struct cxgbit_np
, kref
);
147 cxgbit_create_server6(struct cxgbit_device
*cdev
, unsigned int stid
,
148 struct cxgbit_np
*cnp
)
150 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
151 &cnp
->com
.local_addr
;
155 pr_debug("%s: dev = %s; stid = %u; sin6_port = %u\n",
156 __func__
, cdev
->lldi
.ports
[0]->name
, stid
, sin6
->sin6_port
);
158 addr_type
= ipv6_addr_type((const struct in6_addr
*)
160 if (addr_type
!= IPV6_ADDR_ANY
) {
161 ret
= cxgb4_clip_get(cdev
->lldi
.ports
[0],
162 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
164 pr_err("Unable to find clip table entry. laddr %pI6. Error:%d.\n",
165 sin6
->sin6_addr
.s6_addr
, ret
);
171 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
173 ret
= cxgb4_create_server6(cdev
->lldi
.ports
[0],
174 stid
, &sin6
->sin6_addr
,
176 cdev
->lldi
.rxq_ids
[0]);
178 ret
= cxgbit_wait_for_reply(cdev
, &cnp
->com
.wr_wait
,
181 ret
= net_xmit_errno(ret
);
186 if (ret
!= -ETIMEDOUT
)
187 cxgb4_clip_release(cdev
->lldi
.ports
[0],
188 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
190 pr_err("create server6 err %d stid %d laddr %pI6 lport %d\n",
191 ret
, stid
, sin6
->sin6_addr
.s6_addr
,
192 ntohs(sin6
->sin6_port
));
199 cxgbit_create_server4(struct cxgbit_device
*cdev
, unsigned int stid
,
200 struct cxgbit_np
*cnp
)
202 struct sockaddr_in
*sin
= (struct sockaddr_in
*)
203 &cnp
->com
.local_addr
;
206 pr_debug("%s: dev = %s; stid = %u; sin_port = %u\n",
207 __func__
, cdev
->lldi
.ports
[0]->name
, stid
, sin
->sin_port
);
210 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
212 ret
= cxgb4_create_server(cdev
->lldi
.ports
[0],
213 stid
, sin
->sin_addr
.s_addr
,
215 cdev
->lldi
.rxq_ids
[0]);
217 ret
= cxgbit_wait_for_reply(cdev
,
221 ret
= net_xmit_errno(ret
);
226 pr_err("create server failed err %d stid %d laddr %pI4 lport %d\n",
227 ret
, stid
, &sin
->sin_addr
, ntohs(sin
->sin_port
));
231 struct cxgbit_device
*cxgbit_find_device(struct net_device
*ndev
, u8
*port_id
)
233 struct cxgbit_device
*cdev
;
236 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
237 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
239 for (i
= 0; i
< lldi
->nports
; i
++) {
240 if (lldi
->ports
[i
] == ndev
) {
251 static struct net_device
*cxgbit_get_real_dev(struct net_device
*ndev
)
253 if (ndev
->priv_flags
& IFF_BONDING
) {
254 pr_err("Bond devices are not supported. Interface:%s\n",
259 if (is_vlan_dev(ndev
))
260 return vlan_dev_real_dev(ndev
);
265 static struct net_device
*cxgbit_ipv4_netdev(__be32 saddr
)
267 struct net_device
*ndev
;
269 ndev
= __ip_dev_find(&init_net
, saddr
, false);
273 return cxgbit_get_real_dev(ndev
);
276 static struct net_device
*cxgbit_ipv6_netdev(struct in6_addr
*addr6
)
278 struct net_device
*ndev
= NULL
;
281 if (IS_ENABLED(CONFIG_IPV6
)) {
282 for_each_netdev_rcu(&init_net
, ndev
)
283 if (ipv6_chk_addr(&init_net
, addr6
, ndev
, 1)) {
290 return cxgbit_get_real_dev(ndev
);
293 static struct cxgbit_device
*cxgbit_find_np_cdev(struct cxgbit_np
*cnp
)
295 struct sockaddr_storage
*sockaddr
= &cnp
->com
.local_addr
;
296 int ss_family
= sockaddr
->ss_family
;
297 struct net_device
*ndev
= NULL
;
298 struct cxgbit_device
*cdev
= NULL
;
301 if (ss_family
== AF_INET
) {
302 struct sockaddr_in
*sin
;
304 sin
= (struct sockaddr_in
*)sockaddr
;
305 ndev
= cxgbit_ipv4_netdev(sin
->sin_addr
.s_addr
);
306 } else if (ss_family
== AF_INET6
) {
307 struct sockaddr_in6
*sin6
;
309 sin6
= (struct sockaddr_in6
*)sockaddr
;
310 ndev
= cxgbit_ipv6_netdev(&sin6
->sin6_addr
);
315 cdev
= cxgbit_find_device(ndev
, NULL
);
321 static bool cxgbit_inaddr_any(struct cxgbit_np
*cnp
)
323 struct sockaddr_storage
*sockaddr
= &cnp
->com
.local_addr
;
324 int ss_family
= sockaddr
->ss_family
;
327 if (ss_family
== AF_INET
) {
328 struct sockaddr_in
*sin
;
330 sin
= (struct sockaddr_in
*)sockaddr
;
331 if (sin
->sin_addr
.s_addr
== htonl(INADDR_ANY
))
333 } else if (ss_family
== AF_INET6
) {
334 struct sockaddr_in6
*sin6
;
336 sin6
= (struct sockaddr_in6
*)sockaddr
;
337 addr_type
= ipv6_addr_type((const struct in6_addr
*)
339 if (addr_type
== IPV6_ADDR_ANY
)
346 __cxgbit_setup_cdev_np(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
349 int ss_family
= cnp
->com
.local_addr
.ss_family
;
351 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
))
354 stid
= cxgb4_alloc_stid(cdev
->lldi
.tids
, ss_family
, cnp
);
358 if (!cxgbit_np_hash_add(cdev
, cnp
, stid
)) {
359 cxgb4_free_stid(cdev
->lldi
.tids
, stid
, ss_family
);
363 if (ss_family
== AF_INET
)
364 ret
= cxgbit_create_server4(cdev
, stid
, cnp
);
366 ret
= cxgbit_create_server6(cdev
, stid
, cnp
);
369 if (ret
!= -ETIMEDOUT
)
370 cxgb4_free_stid(cdev
->lldi
.tids
, stid
,
372 cxgbit_np_hash_del(cdev
, cnp
);
378 static int cxgbit_setup_cdev_np(struct cxgbit_np
*cnp
)
380 struct cxgbit_device
*cdev
;
383 mutex_lock(&cdev_list_lock
);
384 cdev
= cxgbit_find_np_cdev(cnp
);
388 if (cxgbit_np_hash_find(cdev
, cnp
) >= 0)
391 if (__cxgbit_setup_cdev_np(cdev
, cnp
))
394 cnp
->com
.cdev
= cdev
;
397 mutex_unlock(&cdev_list_lock
);
401 static int cxgbit_setup_all_np(struct cxgbit_np
*cnp
)
403 struct cxgbit_device
*cdev
;
407 mutex_lock(&cdev_list_lock
);
408 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
409 if (cxgbit_np_hash_find(cdev
, cnp
) >= 0) {
410 mutex_unlock(&cdev_list_lock
);
415 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
416 ret
= __cxgbit_setup_cdev_np(cdev
, cnp
);
417 if (ret
== -ETIMEDOUT
)
423 mutex_unlock(&cdev_list_lock
);
425 return count
? 0 : -1;
428 int cxgbit_setup_np(struct iscsi_np
*np
, struct sockaddr_storage
*ksockaddr
)
430 struct cxgbit_np
*cnp
;
433 if ((ksockaddr
->ss_family
!= AF_INET
) &&
434 (ksockaddr
->ss_family
!= AF_INET6
))
437 cnp
= kzalloc(sizeof(*cnp
), GFP_KERNEL
);
441 init_waitqueue_head(&cnp
->accept_wait
);
442 init_completion(&cnp
->com
.wr_wait
.completion
);
443 init_completion(&cnp
->accept_comp
);
444 INIT_LIST_HEAD(&cnp
->np_accept_list
);
445 spin_lock_init(&cnp
->np_accept_lock
);
446 kref_init(&cnp
->kref
);
447 memcpy(&np
->np_sockaddr
, ksockaddr
,
448 sizeof(struct sockaddr_storage
));
449 memcpy(&cnp
->com
.local_addr
, &np
->np_sockaddr
,
450 sizeof(cnp
->com
.local_addr
));
453 cnp
->com
.cdev
= NULL
;
455 if (cxgbit_inaddr_any(cnp
))
456 ret
= cxgbit_setup_all_np(cnp
);
458 ret
= cxgbit_setup_cdev_np(cnp
);
465 np
->np_context
= cnp
;
466 cnp
->com
.state
= CSK_STATE_LISTEN
;
471 cxgbit_set_conn_info(struct iscsi_np
*np
, struct iscsi_conn
*conn
,
472 struct cxgbit_sock
*csk
)
474 conn
->login_family
= np
->np_sockaddr
.ss_family
;
475 conn
->login_sockaddr
= csk
->com
.remote_addr
;
476 conn
->local_sockaddr
= csk
->com
.local_addr
;
479 int cxgbit_accept_np(struct iscsi_np
*np
, struct iscsi_conn
*conn
)
481 struct cxgbit_np
*cnp
= np
->np_context
;
482 struct cxgbit_sock
*csk
;
486 ret
= wait_for_completion_interruptible(&cnp
->accept_comp
);
490 spin_lock_bh(&np
->np_thread_lock
);
491 if (np
->np_thread_state
>= ISCSI_NP_THREAD_RESET
) {
492 spin_unlock_bh(&np
->np_thread_lock
);
494 * No point in stalling here when np_thread
495 * is in state RESET/SHUTDOWN/EXIT - bail
499 spin_unlock_bh(&np
->np_thread_lock
);
501 spin_lock_bh(&cnp
->np_accept_lock
);
502 if (list_empty(&cnp
->np_accept_list
)) {
503 spin_unlock_bh(&cnp
->np_accept_lock
);
507 csk
= list_first_entry(&cnp
->np_accept_list
,
511 list_del_init(&csk
->accept_node
);
512 spin_unlock_bh(&cnp
->np_accept_lock
);
516 cxgbit_set_conn_info(np
, conn
, csk
);
521 __cxgbit_free_cdev_np(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
526 stid
= cxgbit_np_hash_del(cdev
, cnp
);
529 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
))
532 if (cnp
->np
->np_sockaddr
.ss_family
== AF_INET6
)
536 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
537 ret
= cxgb4_remove_server(cdev
->lldi
.ports
[0], stid
,
538 cdev
->lldi
.rxq_ids
[0], ipv6
);
541 ret
= net_xmit_errno(ret
);
548 ret
= cxgbit_wait_for_reply(cdev
, &cnp
->com
.wr_wait
,
550 if (ret
== -ETIMEDOUT
)
553 if (ipv6
&& cnp
->com
.cdev
) {
554 struct sockaddr_in6
*sin6
;
556 sin6
= (struct sockaddr_in6
*)&cnp
->com
.local_addr
;
557 cxgb4_clip_release(cdev
->lldi
.ports
[0],
558 (const u32
*)&sin6
->sin6_addr
.s6_addr
,
562 cxgb4_free_stid(cdev
->lldi
.tids
, stid
,
563 cnp
->com
.local_addr
.ss_family
);
567 static void cxgbit_free_all_np(struct cxgbit_np
*cnp
)
569 struct cxgbit_device
*cdev
;
572 mutex_lock(&cdev_list_lock
);
573 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
574 ret
= __cxgbit_free_cdev_np(cdev
, cnp
);
575 if (ret
== -ETIMEDOUT
)
578 mutex_unlock(&cdev_list_lock
);
581 static void cxgbit_free_cdev_np(struct cxgbit_np
*cnp
)
583 struct cxgbit_device
*cdev
;
586 mutex_lock(&cdev_list_lock
);
587 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
588 if (cdev
== cnp
->com
.cdev
) {
596 __cxgbit_free_cdev_np(cdev
, cnp
);
598 mutex_unlock(&cdev_list_lock
);
601 void cxgbit_free_np(struct iscsi_np
*np
)
603 struct cxgbit_np
*cnp
= np
->np_context
;
605 cnp
->com
.state
= CSK_STATE_DEAD
;
607 cxgbit_free_cdev_np(cnp
);
609 cxgbit_free_all_np(cnp
);
611 np
->np_context
= NULL
;
615 static void cxgbit_send_halfclose(struct cxgbit_sock
*csk
)
618 u32 len
= roundup(sizeof(struct cpl_close_con_req
), 16);
620 skb
= alloc_skb(len
, GFP_ATOMIC
);
624 cxgb_mk_close_con_req(skb
, len
, csk
->tid
, csk
->txq_idx
,
627 cxgbit_skcb_flags(skb
) |= SKCBF_TX_FLAG_COMPL
;
628 __skb_queue_tail(&csk
->txq
, skb
);
629 cxgbit_push_tx_frames(csk
);
632 static void cxgbit_arp_failure_discard(void *handle
, struct sk_buff
*skb
)
634 pr_debug("%s cxgbit_device %p\n", __func__
, handle
);
638 static void cxgbit_abort_arp_failure(void *handle
, struct sk_buff
*skb
)
640 struct cxgbit_device
*cdev
= handle
;
641 struct cpl_abort_req
*req
= cplhdr(skb
);
643 pr_debug("%s cdev %p\n", __func__
, cdev
);
644 req
->cmd
= CPL_ABORT_NO_RST
;
645 cxgbit_ofld_send(cdev
, skb
);
648 static int cxgbit_send_abort_req(struct cxgbit_sock
*csk
)
651 u32 len
= roundup(sizeof(struct cpl_abort_req
), 16);
653 pr_debug("%s: csk %p tid %u; state %d\n",
654 __func__
, csk
, csk
->tid
, csk
->com
.state
);
656 __skb_queue_purge(&csk
->txq
);
658 if (!test_and_set_bit(CSK_TX_DATA_SENT
, &csk
->com
.flags
))
659 cxgbit_send_tx_flowc_wr(csk
);
661 skb
= __skb_dequeue(&csk
->skbq
);
662 cxgb_mk_abort_req(skb
, len
, csk
->tid
, csk
->txq_idx
,
663 csk
->com
.cdev
, cxgbit_abort_arp_failure
);
665 return cxgbit_l2t_send(csk
->com
.cdev
, skb
, csk
->l2t
);
668 void cxgbit_free_conn(struct iscsi_conn
*conn
)
670 struct cxgbit_sock
*csk
= conn
->context
;
671 bool release
= false;
673 pr_debug("%s: state %d\n",
674 __func__
, csk
->com
.state
);
676 spin_lock_bh(&csk
->lock
);
677 switch (csk
->com
.state
) {
678 case CSK_STATE_ESTABLISHED
:
679 if (conn
->conn_state
== TARG_CONN_STATE_IN_LOGOUT
) {
680 csk
->com
.state
= CSK_STATE_CLOSING
;
681 cxgbit_send_halfclose(csk
);
683 csk
->com
.state
= CSK_STATE_ABORTING
;
684 cxgbit_send_abort_req(csk
);
687 case CSK_STATE_CLOSING
:
688 csk
->com
.state
= CSK_STATE_MORIBUND
;
689 cxgbit_send_halfclose(csk
);
695 pr_err("%s: csk %p; state %d\n",
696 __func__
, csk
, csk
->com
.state
);
698 spin_unlock_bh(&csk
->lock
);
704 static void cxgbit_set_emss(struct cxgbit_sock
*csk
, u16 opt
)
706 csk
->emss
= csk
->com
.cdev
->lldi
.mtus
[TCPOPT_MSS_G(opt
)] -
707 ((csk
->com
.remote_addr
.ss_family
== AF_INET
) ?
708 sizeof(struct iphdr
) : sizeof(struct ipv6hdr
)) -
709 sizeof(struct tcphdr
);
710 csk
->mss
= csk
->emss
;
711 if (TCPOPT_TSTAMP_G(opt
))
712 csk
->emss
-= round_up(TCPOLEN_TIMESTAMP
, 4);
716 pr_info("Warning: misaligned mtu idx %u mss %u emss=%u\n",
717 TCPOPT_MSS_G(opt
), csk
->mss
, csk
->emss
);
718 pr_debug("%s mss_idx %u mss %u emss=%u\n", __func__
, TCPOPT_MSS_G(opt
),
719 csk
->mss
, csk
->emss
);
722 static void cxgbit_free_skb(struct cxgbit_sock
*csk
)
726 __skb_queue_purge(&csk
->txq
);
727 __skb_queue_purge(&csk
->rxq
);
728 __skb_queue_purge(&csk
->backlogq
);
729 __skb_queue_purge(&csk
->ppodq
);
730 __skb_queue_purge(&csk
->skbq
);
732 while ((skb
= cxgbit_sock_dequeue_wr(csk
)))
735 __kfree_skb(csk
->lro_hskb
);
738 void _cxgbit_free_csk(struct kref
*kref
)
740 struct cxgbit_sock
*csk
;
741 struct cxgbit_device
*cdev
;
743 csk
= container_of(kref
, struct cxgbit_sock
, kref
);
745 pr_debug("%s csk %p state %d\n", __func__
, csk
, csk
->com
.state
);
747 if (csk
->com
.local_addr
.ss_family
== AF_INET6
) {
748 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
749 &csk
->com
.local_addr
;
750 cxgb4_clip_release(csk
->com
.cdev
->lldi
.ports
[0],
752 &sin6
->sin6_addr
.s6_addr
, 1);
755 cxgb4_remove_tid(csk
->com
.cdev
->lldi
.tids
, 0, csk
->tid
,
756 csk
->com
.local_addr
.ss_family
);
757 dst_release(csk
->dst
);
758 cxgb4_l2t_release(csk
->l2t
);
760 cdev
= csk
->com
.cdev
;
761 spin_lock_bh(&cdev
->cskq
.lock
);
762 list_del(&csk
->list
);
763 spin_unlock_bh(&cdev
->cskq
.lock
);
765 cxgbit_free_skb(csk
);
766 cxgbit_put_cdev(cdev
);
771 static void cxgbit_set_tcp_window(struct cxgbit_sock
*csk
, struct port_info
*pi
)
773 unsigned int linkspeed
;
776 linkspeed
= pi
->link_cfg
.speed
;
777 scale
= linkspeed
/ SPEED_10000
;
779 #define CXGBIT_10G_RCV_WIN (256 * 1024)
780 csk
->rcv_win
= CXGBIT_10G_RCV_WIN
;
782 csk
->rcv_win
*= scale
;
784 #define CXGBIT_10G_SND_WIN (256 * 1024)
785 csk
->snd_win
= CXGBIT_10G_SND_WIN
;
787 csk
->snd_win
*= scale
;
789 pr_debug("%s snd_win %d rcv_win %d\n",
790 __func__
, csk
->snd_win
, csk
->rcv_win
);
793 #ifdef CONFIG_CHELSIO_T4_DCB
794 static u8
cxgbit_get_iscsi_dcb_state(struct net_device
*ndev
)
796 return ndev
->dcbnl_ops
->getstate(ndev
);
799 static int cxgbit_select_priority(int pri_mask
)
804 return (ffs(pri_mask
) - 1);
807 static u8
cxgbit_get_iscsi_dcb_priority(struct net_device
*ndev
, u16 local_port
)
812 struct dcb_app iscsi_dcb_app
= {
813 .protocol
= local_port
816 ret
= (int)ndev
->dcbnl_ops
->getcap(ndev
, DCB_CAP_ATTR_DCBX
, &caps
);
821 if (caps
& DCB_CAP_DCBX_VER_IEEE
) {
822 iscsi_dcb_app
.selector
= IEEE_8021QAZ_APP_SEL_ANY
;
824 ret
= dcb_ieee_getapp_mask(ndev
, &iscsi_dcb_app
);
826 } else if (caps
& DCB_CAP_DCBX_VER_CEE
) {
827 iscsi_dcb_app
.selector
= DCB_APP_IDTYPE_PORTNUM
;
829 ret
= dcb_getapp(ndev
, &iscsi_dcb_app
);
832 pr_info("iSCSI priority is set to %u\n", cxgbit_select_priority(ret
));
834 return cxgbit_select_priority(ret
);
839 cxgbit_offload_init(struct cxgbit_sock
*csk
, int iptype
, __u8
*peer_ip
,
840 u16 local_port
, struct dst_entry
*dst
,
841 struct cxgbit_device
*cdev
)
845 struct net_device
*ndev
;
846 u16 rxq_idx
, port_id
;
847 #ifdef CONFIG_CHELSIO_T4_DCB
851 n
= dst_neigh_lookup(dst
, peer_ip
);
857 if (n
->dev
->flags
& IFF_LOOPBACK
) {
859 ndev
= cxgbit_ipv4_netdev(*(__be32
*)peer_ip
);
860 else if (IS_ENABLED(CONFIG_IPV6
))
861 ndev
= cxgbit_ipv6_netdev((struct in6_addr
*)peer_ip
);
870 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
,
874 csk
->mtu
= ndev
->mtu
;
875 csk
->tx_chan
= cxgb4_port_chan(ndev
);
876 csk
->smac_idx
= cxgb4_tp_smt_idx(cdev
->lldi
.adapter_type
,
877 cxgb4_port_viid(ndev
));
878 step
= cdev
->lldi
.ntxq
/
880 csk
->txq_idx
= cxgb4_port_idx(ndev
) * step
;
881 step
= cdev
->lldi
.nrxq
/
883 csk
->ctrlq_idx
= cxgb4_port_idx(ndev
);
884 csk
->rss_qid
= cdev
->lldi
.rxq_ids
[
885 cxgb4_port_idx(ndev
) * step
];
886 csk
->port_id
= cxgb4_port_idx(ndev
);
887 cxgbit_set_tcp_window(csk
,
888 (struct port_info
*)netdev_priv(ndev
));
890 ndev
= cxgbit_get_real_dev(n
->dev
);
896 #ifdef CONFIG_CHELSIO_T4_DCB
897 if (cxgbit_get_iscsi_dcb_state(ndev
))
898 priority
= cxgbit_get_iscsi_dcb_priority(ndev
,
901 csk
->dcb_priority
= priority
;
903 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
, n
, ndev
, priority
);
905 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
, n
, ndev
, 0);
909 port_id
= cxgb4_port_idx(ndev
);
910 csk
->mtu
= dst_mtu(dst
);
911 csk
->tx_chan
= cxgb4_port_chan(ndev
);
912 csk
->smac_idx
= cxgb4_tp_smt_idx(cdev
->lldi
.adapter_type
,
913 cxgb4_port_viid(ndev
));
914 step
= cdev
->lldi
.ntxq
/
916 csk
->txq_idx
= (port_id
* step
) +
917 (cdev
->selectq
[port_id
][0]++ % step
);
918 csk
->ctrlq_idx
= cxgb4_port_idx(ndev
);
919 step
= cdev
->lldi
.nrxq
/
921 rxq_idx
= (port_id
* step
) +
922 (cdev
->selectq
[port_id
][1]++ % step
);
923 csk
->rss_qid
= cdev
->lldi
.rxq_ids
[rxq_idx
];
924 csk
->port_id
= port_id
;
925 cxgbit_set_tcp_window(csk
,
926 (struct port_info
*)netdev_priv(ndev
));
935 int cxgbit_ofld_send(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
939 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
941 pr_err("%s - device not up - dropping\n", __func__
);
945 ret
= cxgb4_ofld_send(cdev
->lldi
.ports
[0], skb
);
948 return ret
< 0 ? ret
: 0;
951 static void cxgbit_release_tid(struct cxgbit_device
*cdev
, u32 tid
)
953 u32 len
= roundup(sizeof(struct cpl_tid_release
), 16);
956 skb
= alloc_skb(len
, GFP_ATOMIC
);
960 cxgb_mk_tid_release(skb
, len
, tid
, 0);
961 cxgbit_ofld_send(cdev
, skb
);
965 cxgbit_l2t_send(struct cxgbit_device
*cdev
, struct sk_buff
*skb
,
966 struct l2t_entry
*l2e
)
970 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
972 pr_err("%s - device not up - dropping\n", __func__
);
976 ret
= cxgb4_l2t_send(cdev
->lldi
.ports
[0], skb
, l2e
);
979 return ret
< 0 ? ret
: 0;
982 static void cxgbit_send_rx_credits(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
984 if (csk
->com
.state
!= CSK_STATE_ESTABLISHED
) {
989 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
993 * CPL connection rx data ack: host ->
994 * Send RX credits through an RX_DATA_ACK CPL message.
995 * Returns the number of credits sent.
997 int cxgbit_rx_data_ack(struct cxgbit_sock
*csk
)
1000 u32 len
= roundup(sizeof(struct cpl_rx_data_ack
), 16);
1003 skb
= alloc_skb(len
, GFP_KERNEL
);
1007 credit_dack
= RX_DACK_CHANGE_F
| RX_DACK_MODE_V(1) |
1008 RX_CREDITS_V(csk
->rx_credits
);
1010 cxgb_mk_rx_data_ack(skb
, len
, csk
->tid
, csk
->ctrlq_idx
,
1013 csk
->rx_credits
= 0;
1015 spin_lock_bh(&csk
->lock
);
1016 if (csk
->lock_owner
) {
1017 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_send_rx_credits
;
1018 __skb_queue_tail(&csk
->backlogq
, skb
);
1019 spin_unlock_bh(&csk
->lock
);
1023 cxgbit_send_rx_credits(csk
, skb
);
1024 spin_unlock_bh(&csk
->lock
);
1029 #define FLOWC_WR_NPARAMS_MIN 9
1030 #define FLOWC_WR_NPARAMS_MAX 11
1031 static int cxgbit_alloc_csk_skb(struct cxgbit_sock
*csk
)
1033 struct sk_buff
*skb
;
1037 flowclen
= offsetof(struct fw_flowc_wr
,
1038 mnemval
[FLOWC_WR_NPARAMS_MAX
]);
1040 len
= max_t(u32
, sizeof(struct cpl_abort_req
),
1041 sizeof(struct cpl_abort_rpl
));
1043 len
= max(len
, flowclen
);
1044 len
= roundup(len
, 16);
1046 for (i
= 0; i
< 3; i
++) {
1047 skb
= alloc_skb(len
, GFP_ATOMIC
);
1050 __skb_queue_tail(&csk
->skbq
, skb
);
1053 skb
= alloc_skb(LRO_SKB_MIN_HEADROOM
, GFP_ATOMIC
);
1057 memset(skb
->data
, 0, LRO_SKB_MIN_HEADROOM
);
1058 csk
->lro_hskb
= skb
;
1062 __skb_queue_purge(&csk
->skbq
);
1067 cxgbit_pass_accept_rpl(struct cxgbit_sock
*csk
, struct cpl_pass_accept_req
*req
)
1069 struct sk_buff
*skb
;
1070 const struct tcphdr
*tcph
;
1071 struct cpl_t5_pass_accept_rpl
*rpl5
;
1072 struct cxgb4_lld_info
*lldi
= &csk
->com
.cdev
->lldi
;
1073 unsigned int len
= roundup(sizeof(*rpl5
), 16);
1074 unsigned int mtu_idx
;
1080 pr_debug("%s csk %p tid %u\n", __func__
, csk
, csk
->tid
);
1082 skb
= alloc_skb(len
, GFP_ATOMIC
);
1084 cxgbit_put_csk(csk
);
1088 rpl5
= __skb_put_zero(skb
, len
);
1090 INIT_TP_WR(rpl5
, csk
->tid
);
1091 OPCODE_TID(rpl5
) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL
,
1093 cxgb_best_mtu(csk
->com
.cdev
->lldi
.mtus
, csk
->mtu
, &mtu_idx
,
1095 (csk
->com
.remote_addr
.ss_family
== AF_INET
) ? 0 : 1);
1096 wscale
= cxgb_compute_wscale(csk
->rcv_win
);
1098 * Specify the largest window that will fit in opt0. The
1099 * remainder will be specified in the rx_data_ack.
1101 win
= csk
->rcv_win
>> 10;
1102 if (win
> RCV_BUFSIZ_M
)
1104 opt0
= TCAM_BYPASS_F
|
1105 WND_SCALE_V(wscale
) |
1106 MSS_IDX_V(mtu_idx
) |
1107 L2T_IDX_V(csk
->l2t
->idx
) |
1108 TX_CHAN_V(csk
->tx_chan
) |
1109 SMAC_SEL_V(csk
->smac_idx
) |
1110 DSCP_V(csk
->tos
>> 2) |
1111 ULP_MODE_V(ULP_MODE_ISCSI
) |
1114 opt2
= RX_CHANNEL_V(0) |
1115 RSS_QUEUE_VALID_F
| RSS_QUEUE_V(csk
->rss_qid
);
1117 if (!is_t5(lldi
->adapter_type
))
1118 opt2
|= RX_FC_DISABLE_F
;
1120 if (req
->tcpopt
.tstamp
)
1121 opt2
|= TSTAMPS_EN_F
;
1122 if (req
->tcpopt
.sack
)
1125 opt2
|= WND_SCALE_EN_F
;
1127 hlen
= ntohl(req
->hdr_len
);
1129 if (is_t5(lldi
->adapter_type
))
1130 tcph
= (struct tcphdr
*)((u8
*)(req
+ 1) +
1131 ETH_HDR_LEN_G(hlen
) + IP_HDR_LEN_G(hlen
));
1133 tcph
= (struct tcphdr
*)((u8
*)(req
+ 1) +
1134 T6_ETH_HDR_LEN_G(hlen
) + T6_IP_HDR_LEN_G(hlen
));
1136 if (tcph
->ece
&& tcph
->cwr
)
1137 opt2
|= CCTRL_ECN_V(1);
1139 opt2
|= RX_COALESCE_V(3);
1140 opt2
|= CONG_CNTRL_V(CONG_ALG_NEWRENO
);
1143 rpl5
->iss
= cpu_to_be32((prandom_u32() & ~7UL) - 1);
1145 opt2
|= T5_OPT_2_VALID_F
;
1147 rpl5
->opt0
= cpu_to_be64(opt0
);
1148 rpl5
->opt2
= cpu_to_be32(opt2
);
1149 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, csk
->ctrlq_idx
);
1150 t4_set_arp_err_handler(skb
, NULL
, cxgbit_arp_failure_discard
);
1151 cxgbit_l2t_send(csk
->com
.cdev
, skb
, csk
->l2t
);
1155 cxgbit_pass_accept_req(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1157 struct cxgbit_sock
*csk
= NULL
;
1158 struct cxgbit_np
*cnp
;
1159 struct cpl_pass_accept_req
*req
= cplhdr(skb
);
1160 unsigned int stid
= PASS_OPEN_TID_G(ntohl(req
->tos_stid
));
1161 struct tid_info
*t
= cdev
->lldi
.tids
;
1162 unsigned int tid
= GET_TID(req
);
1163 u16 peer_mss
= ntohs(req
->tcpopt
.mss
);
1164 unsigned short hdrs
;
1166 struct dst_entry
*dst
;
1167 __u8 local_ip
[16], peer_ip
[16];
1168 __be16 local_port
, peer_port
;
1172 pr_debug("%s: cdev = %p; stid = %u; tid = %u\n",
1173 __func__
, cdev
, stid
, tid
);
1175 cnp
= lookup_stid(t
, stid
);
1177 pr_err("%s connect request on invalid stid %d\n",
1182 if (cnp
->com
.state
!= CSK_STATE_LISTEN
) {
1183 pr_err("%s - listening parent not in CSK_STATE_LISTEN\n",
1188 csk
= lookup_tid(t
, tid
);
1190 pr_err("%s csk not null tid %u\n",
1195 cxgb_get_4tuple(req
, cdev
->lldi
.adapter_type
, &iptype
, local_ip
,
1196 peer_ip
, &local_port
, &peer_port
);
1198 /* Find output route */
1200 pr_debug("%s parent sock %p tid %u laddr %pI4 raddr %pI4 "
1201 "lport %d rport %d peer_mss %d\n"
1202 , __func__
, cnp
, tid
,
1203 local_ip
, peer_ip
, ntohs(local_port
),
1204 ntohs(peer_port
), peer_mss
);
1205 dst
= cxgb_find_route(&cdev
->lldi
, cxgbit_get_real_dev
,
1206 *(__be32
*)local_ip
,
1208 local_port
, peer_port
,
1209 PASS_OPEN_TOS_G(ntohl(req
->tos_stid
)));
1211 pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 "
1212 "lport %d rport %d peer_mss %d\n"
1213 , __func__
, cnp
, tid
,
1214 local_ip
, peer_ip
, ntohs(local_port
),
1215 ntohs(peer_port
), peer_mss
);
1216 dst
= cxgb_find_route6(&cdev
->lldi
, cxgbit_get_real_dev
,
1218 local_port
, peer_port
,
1219 PASS_OPEN_TOS_G(ntohl(req
->tos_stid
)),
1220 ((struct sockaddr_in6
*)
1221 &cnp
->com
.local_addr
)->sin6_scope_id
);
1224 pr_err("%s - failed to find dst entry!\n",
1229 csk
= kzalloc(sizeof(*csk
), GFP_ATOMIC
);
1235 ret
= cxgbit_offload_init(csk
, iptype
, peer_ip
, ntohs(local_port
),
1238 pr_err("%s - failed to allocate l2t entry!\n",
1245 kref_init(&csk
->kref
);
1246 init_completion(&csk
->com
.wr_wait
.completion
);
1248 INIT_LIST_HEAD(&csk
->accept_node
);
1250 hdrs
= (iptype
== 4 ? sizeof(struct iphdr
) : sizeof(struct ipv6hdr
)) +
1251 sizeof(struct tcphdr
) + (req
->tcpopt
.tstamp
? 12 : 0);
1252 if (peer_mss
&& csk
->mtu
> (peer_mss
+ hdrs
))
1253 csk
->mtu
= peer_mss
+ hdrs
;
1255 csk
->com
.state
= CSK_STATE_CONNECTING
;
1256 csk
->com
.cdev
= cdev
;
1258 csk
->tos
= PASS_OPEN_TOS_G(ntohl(req
->tos_stid
));
1261 csk
->wr_cred
= cdev
->lldi
.wr_cred
-
1262 DIV_ROUND_UP(sizeof(struct cpl_abort_req
), 16);
1263 csk
->wr_max_cred
= csk
->wr_cred
;
1264 csk
->wr_una_cred
= 0;
1267 struct sockaddr_in
*sin
= (struct sockaddr_in
*)
1268 &csk
->com
.local_addr
;
1269 sin
->sin_family
= AF_INET
;
1270 sin
->sin_port
= local_port
;
1271 sin
->sin_addr
.s_addr
= *(__be32
*)local_ip
;
1273 sin
= (struct sockaddr_in
*)&csk
->com
.remote_addr
;
1274 sin
->sin_family
= AF_INET
;
1275 sin
->sin_port
= peer_port
;
1276 sin
->sin_addr
.s_addr
= *(__be32
*)peer_ip
;
1278 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
1279 &csk
->com
.local_addr
;
1281 sin6
->sin6_family
= PF_INET6
;
1282 sin6
->sin6_port
= local_port
;
1283 memcpy(sin6
->sin6_addr
.s6_addr
, local_ip
, 16);
1284 cxgb4_clip_get(cdev
->lldi
.ports
[0],
1285 (const u32
*)&sin6
->sin6_addr
.s6_addr
,
1288 sin6
= (struct sockaddr_in6
*)&csk
->com
.remote_addr
;
1289 sin6
->sin6_family
= PF_INET6
;
1290 sin6
->sin6_port
= peer_port
;
1291 memcpy(sin6
->sin6_addr
.s6_addr
, peer_ip
, 16);
1294 skb_queue_head_init(&csk
->rxq
);
1295 skb_queue_head_init(&csk
->txq
);
1296 skb_queue_head_init(&csk
->ppodq
);
1297 skb_queue_head_init(&csk
->backlogq
);
1298 skb_queue_head_init(&csk
->skbq
);
1299 cxgbit_sock_reset_wr_list(csk
);
1300 spin_lock_init(&csk
->lock
);
1301 init_waitqueue_head(&csk
->waitq
);
1302 init_waitqueue_head(&csk
->ack_waitq
);
1303 csk
->lock_owner
= false;
1305 if (cxgbit_alloc_csk_skb(csk
)) {
1311 cxgbit_get_cdev(cdev
);
1313 spin_lock(&cdev
->cskq
.lock
);
1314 list_add_tail(&csk
->list
, &cdev
->cskq
.list
);
1315 spin_unlock(&cdev
->cskq
.lock
);
1316 cxgb4_insert_tid(t
, csk
, tid
, csk
->com
.local_addr
.ss_family
);
1317 cxgbit_pass_accept_rpl(csk
, req
);
1321 cxgbit_release_tid(cdev
, tid
);
1327 cxgbit_tx_flowc_wr_credits(struct cxgbit_sock
*csk
, u32
*nparamsp
,
1330 u32 nparams
, flowclen16
, flowclen
;
1332 nparams
= FLOWC_WR_NPARAMS_MIN
;
1334 if (csk
->snd_wscale
)
1337 #ifdef CONFIG_CHELSIO_T4_DCB
1340 flowclen
= offsetof(struct fw_flowc_wr
, mnemval
[nparams
]);
1341 flowclen16
= DIV_ROUND_UP(flowclen
, 16);
1342 flowclen
= flowclen16
* 16;
1344 * Return the number of 16-byte credits used by the flowc request.
1345 * Pass back the nparams and actual flowc length if requested.
1348 *nparamsp
= nparams
;
1350 *flowclenp
= flowclen
;
1354 u32
cxgbit_send_tx_flowc_wr(struct cxgbit_sock
*csk
)
1356 struct cxgbit_device
*cdev
= csk
->com
.cdev
;
1357 struct fw_flowc_wr
*flowc
;
1358 u32 nparams
, flowclen16
, flowclen
;
1359 struct sk_buff
*skb
;
1362 #ifdef CONFIG_CHELSIO_T4_DCB
1363 u16 vlan
= ((struct l2t_entry
*)csk
->l2t
)->vlan
;
1366 flowclen16
= cxgbit_tx_flowc_wr_credits(csk
, &nparams
, &flowclen
);
1368 skb
= __skb_dequeue(&csk
->skbq
);
1369 flowc
= __skb_put_zero(skb
, flowclen
);
1371 flowc
->op_to_nparams
= cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR
) |
1372 FW_FLOWC_WR_NPARAMS_V(nparams
));
1373 flowc
->flowid_len16
= cpu_to_be32(FW_WR_LEN16_V(flowclen16
) |
1374 FW_WR_FLOWID_V(csk
->tid
));
1375 flowc
->mnemval
[0].mnemonic
= FW_FLOWC_MNEM_PFNVFN
;
1376 flowc
->mnemval
[0].val
= cpu_to_be32(FW_PFVF_CMD_PFN_V
1377 (csk
->com
.cdev
->lldi
.pf
));
1378 flowc
->mnemval
[1].mnemonic
= FW_FLOWC_MNEM_CH
;
1379 flowc
->mnemval
[1].val
= cpu_to_be32(csk
->tx_chan
);
1380 flowc
->mnemval
[2].mnemonic
= FW_FLOWC_MNEM_PORT
;
1381 flowc
->mnemval
[2].val
= cpu_to_be32(csk
->tx_chan
);
1382 flowc
->mnemval
[3].mnemonic
= FW_FLOWC_MNEM_IQID
;
1383 flowc
->mnemval
[3].val
= cpu_to_be32(csk
->rss_qid
);
1384 flowc
->mnemval
[4].mnemonic
= FW_FLOWC_MNEM_SNDNXT
;
1385 flowc
->mnemval
[4].val
= cpu_to_be32(csk
->snd_nxt
);
1386 flowc
->mnemval
[5].mnemonic
= FW_FLOWC_MNEM_RCVNXT
;
1387 flowc
->mnemval
[5].val
= cpu_to_be32(csk
->rcv_nxt
);
1388 flowc
->mnemval
[6].mnemonic
= FW_FLOWC_MNEM_SNDBUF
;
1389 flowc
->mnemval
[6].val
= cpu_to_be32(csk
->snd_win
);
1390 flowc
->mnemval
[7].mnemonic
= FW_FLOWC_MNEM_MSS
;
1391 flowc
->mnemval
[7].val
= cpu_to_be32(csk
->emss
);
1393 flowc
->mnemval
[8].mnemonic
= FW_FLOWC_MNEM_TXDATAPLEN_MAX
;
1394 if (test_bit(CDEV_ISO_ENABLE
, &cdev
->flags
))
1395 flowc
->mnemval
[8].val
= cpu_to_be32(CXGBIT_MAX_ISO_PAYLOAD
);
1397 flowc
->mnemval
[8].val
= cpu_to_be32(16384);
1401 if (csk
->snd_wscale
) {
1402 flowc
->mnemval
[index
].mnemonic
= FW_FLOWC_MNEM_RCV_SCALE
;
1403 flowc
->mnemval
[index
].val
= cpu_to_be32(csk
->snd_wscale
);
1407 #ifdef CONFIG_CHELSIO_T4_DCB
1408 flowc
->mnemval
[index
].mnemonic
= FW_FLOWC_MNEM_DCBPRIO
;
1409 if (vlan
== VLAN_NONE
) {
1410 pr_warn("csk %u without VLAN Tag on DCB Link\n", csk
->tid
);
1411 flowc
->mnemval
[index
].val
= cpu_to_be32(0);
1413 flowc
->mnemval
[index
].val
= cpu_to_be32(
1414 (vlan
& VLAN_PRIO_MASK
) >> VLAN_PRIO_SHIFT
);
1417 pr_debug("%s: csk %p; tx_chan = %u; rss_qid = %u; snd_seq = %u;"
1418 " rcv_seq = %u; snd_win = %u; emss = %u\n",
1419 __func__
, csk
, csk
->tx_chan
, csk
->rss_qid
, csk
->snd_nxt
,
1420 csk
->rcv_nxt
, csk
->snd_win
, csk
->emss
);
1421 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->txq_idx
);
1422 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1426 int cxgbit_setup_conn_digest(struct cxgbit_sock
*csk
)
1428 struct sk_buff
*skb
;
1429 struct cpl_set_tcb_field
*req
;
1430 u8 hcrc
= csk
->submode
& CXGBIT_SUBMODE_HCRC
;
1431 u8 dcrc
= csk
->submode
& CXGBIT_SUBMODE_DCRC
;
1432 unsigned int len
= roundup(sizeof(*req
), 16);
1435 skb
= alloc_skb(len
, GFP_KERNEL
);
1439 /* set up ulp submode */
1440 req
= __skb_put_zero(skb
, len
);
1442 INIT_TP_WR(req
, csk
->tid
);
1443 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, csk
->tid
));
1444 req
->reply_ctrl
= htons(NO_REPLY_V(0) | QUEUENO_V(csk
->rss_qid
));
1445 req
->word_cookie
= htons(0);
1446 req
->mask
= cpu_to_be64(0x3 << 4);
1447 req
->val
= cpu_to_be64(((hcrc
? ULP_CRC_HEADER
: 0) |
1448 (dcrc
? ULP_CRC_DATA
: 0)) << 4);
1449 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->ctrlq_idx
);
1451 cxgbit_get_csk(csk
);
1452 cxgbit_init_wr_wait(&csk
->com
.wr_wait
);
1454 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1456 ret
= cxgbit_wait_for_reply(csk
->com
.cdev
,
1458 csk
->tid
, 5, __func__
);
1465 int cxgbit_setup_conn_pgidx(struct cxgbit_sock
*csk
, u32 pg_idx
)
1467 struct sk_buff
*skb
;
1468 struct cpl_set_tcb_field
*req
;
1469 unsigned int len
= roundup(sizeof(*req
), 16);
1472 skb
= alloc_skb(len
, GFP_KERNEL
);
1476 req
= __skb_put_zero(skb
, len
);
1478 INIT_TP_WR(req
, csk
->tid
);
1479 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, csk
->tid
));
1480 req
->reply_ctrl
= htons(NO_REPLY_V(0) | QUEUENO_V(csk
->rss_qid
));
1481 req
->word_cookie
= htons(0);
1482 req
->mask
= cpu_to_be64(0x3 << 8);
1483 req
->val
= cpu_to_be64(pg_idx
<< 8);
1484 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->ctrlq_idx
);
1486 cxgbit_get_csk(csk
);
1487 cxgbit_init_wr_wait(&csk
->com
.wr_wait
);
1489 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1491 ret
= cxgbit_wait_for_reply(csk
->com
.cdev
,
1493 csk
->tid
, 5, __func__
);
1501 cxgbit_pass_open_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1503 struct cpl_pass_open_rpl
*rpl
= cplhdr(skb
);
1504 struct tid_info
*t
= cdev
->lldi
.tids
;
1505 unsigned int stid
= GET_TID(rpl
);
1506 struct cxgbit_np
*cnp
= lookup_stid(t
, stid
);
1508 pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1509 __func__
, cnp
, stid
, rpl
->status
);
1512 pr_info("%s stid %d lookup failure\n", __func__
, stid
);
1516 cxgbit_wake_up(&cnp
->com
.wr_wait
, __func__
, rpl
->status
);
1517 cxgbit_put_cnp(cnp
);
1523 cxgbit_close_listsrv_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1525 struct cpl_close_listsvr_rpl
*rpl
= cplhdr(skb
);
1526 struct tid_info
*t
= cdev
->lldi
.tids
;
1527 unsigned int stid
= GET_TID(rpl
);
1528 struct cxgbit_np
*cnp
= lookup_stid(t
, stid
);
1530 pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1531 __func__
, cnp
, stid
, rpl
->status
);
1534 pr_info("%s stid %d lookup failure\n", __func__
, stid
);
1538 cxgbit_wake_up(&cnp
->com
.wr_wait
, __func__
, rpl
->status
);
1539 cxgbit_put_cnp(cnp
);
1545 cxgbit_pass_establish(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1547 struct cpl_pass_establish
*req
= cplhdr(skb
);
1548 struct tid_info
*t
= cdev
->lldi
.tids
;
1549 unsigned int tid
= GET_TID(req
);
1550 struct cxgbit_sock
*csk
;
1551 struct cxgbit_np
*cnp
;
1552 u16 tcp_opt
= be16_to_cpu(req
->tcp_opt
);
1553 u32 snd_isn
= be32_to_cpu(req
->snd_isn
);
1554 u32 rcv_isn
= be32_to_cpu(req
->rcv_isn
);
1556 csk
= lookup_tid(t
, tid
);
1557 if (unlikely(!csk
)) {
1558 pr_err("can't find connection for tid %u.\n", tid
);
1563 pr_debug("%s: csk %p; tid %u; cnp %p\n",
1564 __func__
, csk
, tid
, cnp
);
1566 csk
->write_seq
= snd_isn
;
1567 csk
->snd_una
= snd_isn
;
1568 csk
->snd_nxt
= snd_isn
;
1570 csk
->rcv_nxt
= rcv_isn
;
1572 if (csk
->rcv_win
> (RCV_BUFSIZ_M
<< 10))
1573 csk
->rx_credits
= (csk
->rcv_win
- (RCV_BUFSIZ_M
<< 10));
1575 csk
->snd_wscale
= TCPOPT_SND_WSCALE_G(tcp_opt
);
1576 cxgbit_set_emss(csk
, tcp_opt
);
1577 dst_confirm(csk
->dst
);
1578 csk
->com
.state
= CSK_STATE_ESTABLISHED
;
1579 spin_lock_bh(&cnp
->np_accept_lock
);
1580 list_add_tail(&csk
->accept_node
, &cnp
->np_accept_list
);
1581 spin_unlock_bh(&cnp
->np_accept_lock
);
1582 complete(&cnp
->accept_comp
);
1587 static void cxgbit_queue_rx_skb(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1589 cxgbit_skcb_flags(skb
) = 0;
1590 spin_lock_bh(&csk
->rxq
.lock
);
1591 __skb_queue_tail(&csk
->rxq
, skb
);
1592 spin_unlock_bh(&csk
->rxq
.lock
);
1593 wake_up(&csk
->waitq
);
1596 static void cxgbit_peer_close(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1598 pr_debug("%s: csk %p; tid %u; state %d\n",
1599 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1601 switch (csk
->com
.state
) {
1602 case CSK_STATE_ESTABLISHED
:
1603 csk
->com
.state
= CSK_STATE_CLOSING
;
1604 cxgbit_queue_rx_skb(csk
, skb
);
1606 case CSK_STATE_CLOSING
:
1607 /* simultaneous close */
1608 csk
->com
.state
= CSK_STATE_MORIBUND
;
1610 case CSK_STATE_MORIBUND
:
1611 csk
->com
.state
= CSK_STATE_DEAD
;
1612 cxgbit_put_csk(csk
);
1614 case CSK_STATE_ABORTING
:
1617 pr_info("%s: cpl_peer_close in bad state %d\n",
1618 __func__
, csk
->com
.state
);
1624 static void cxgbit_close_con_rpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1626 pr_debug("%s: csk %p; tid %u; state %d\n",
1627 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1629 switch (csk
->com
.state
) {
1630 case CSK_STATE_CLOSING
:
1631 csk
->com
.state
= CSK_STATE_MORIBUND
;
1633 case CSK_STATE_MORIBUND
:
1634 csk
->com
.state
= CSK_STATE_DEAD
;
1635 cxgbit_put_csk(csk
);
1637 case CSK_STATE_ABORTING
:
1638 case CSK_STATE_DEAD
:
1641 pr_info("%s: cpl_close_con_rpl in bad state %d\n",
1642 __func__
, csk
->com
.state
);
1648 static void cxgbit_abort_req_rss(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1650 struct cpl_abort_req_rss
*hdr
= cplhdr(skb
);
1651 unsigned int tid
= GET_TID(hdr
);
1652 struct sk_buff
*rpl_skb
;
1653 bool release
= false;
1654 bool wakeup_thread
= false;
1655 u32 len
= roundup(sizeof(struct cpl_abort_rpl
), 16);
1657 pr_debug("%s: csk %p; tid %u; state %d\n",
1658 __func__
, csk
, tid
, csk
->com
.state
);
1660 if (cxgb_is_neg_adv(hdr
->status
)) {
1661 pr_err("%s: got neg advise %d on tid %u\n",
1662 __func__
, hdr
->status
, tid
);
1666 switch (csk
->com
.state
) {
1667 case CSK_STATE_CONNECTING
:
1668 case CSK_STATE_MORIBUND
:
1669 csk
->com
.state
= CSK_STATE_DEAD
;
1672 case CSK_STATE_ESTABLISHED
:
1673 csk
->com
.state
= CSK_STATE_DEAD
;
1674 wakeup_thread
= true;
1676 case CSK_STATE_CLOSING
:
1677 csk
->com
.state
= CSK_STATE_DEAD
;
1681 case CSK_STATE_ABORTING
:
1684 pr_info("%s: cpl_abort_req_rss in bad state %d\n",
1685 __func__
, csk
->com
.state
);
1686 csk
->com
.state
= CSK_STATE_DEAD
;
1689 __skb_queue_purge(&csk
->txq
);
1691 if (!test_and_set_bit(CSK_TX_DATA_SENT
, &csk
->com
.flags
))
1692 cxgbit_send_tx_flowc_wr(csk
);
1694 rpl_skb
= __skb_dequeue(&csk
->skbq
);
1696 cxgb_mk_abort_rpl(rpl_skb
, len
, csk
->tid
, csk
->txq_idx
);
1697 cxgbit_ofld_send(csk
->com
.cdev
, rpl_skb
);
1699 if (wakeup_thread
) {
1700 cxgbit_queue_rx_skb(csk
, skb
);
1705 cxgbit_put_csk(csk
);
1710 static void cxgbit_abort_rpl_rss(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1712 pr_debug("%s: csk %p; tid %u; state %d\n",
1713 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1715 switch (csk
->com
.state
) {
1716 case CSK_STATE_ABORTING
:
1717 csk
->com
.state
= CSK_STATE_DEAD
;
1718 cxgbit_put_csk(csk
);
1721 pr_info("%s: cpl_abort_rpl_rss in state %d\n",
1722 __func__
, csk
->com
.state
);
1728 static bool cxgbit_credit_err(const struct cxgbit_sock
*csk
)
1730 const struct sk_buff
*skb
= csk
->wr_pending_head
;
1733 if (unlikely(csk
->wr_cred
> csk
->wr_max_cred
)) {
1734 pr_err("csk 0x%p, tid %u, credit %u > %u\n",
1735 csk
, csk
->tid
, csk
->wr_cred
, csk
->wr_max_cred
);
1740 credit
+= (__force u32
)skb
->csum
;
1741 skb
= cxgbit_skcb_tx_wr_next(skb
);
1744 if (unlikely((csk
->wr_cred
+ credit
) != csk
->wr_max_cred
)) {
1745 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1746 csk
, csk
->tid
, csk
->wr_cred
,
1747 credit
, csk
->wr_max_cred
);
1755 static void cxgbit_fw4_ack(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1757 struct cpl_fw4_ack
*rpl
= (struct cpl_fw4_ack
*)cplhdr(skb
);
1758 u32 credits
= rpl
->credits
;
1759 u32 snd_una
= ntohl(rpl
->snd_una
);
1761 csk
->wr_cred
+= credits
;
1762 if (csk
->wr_una_cred
> (csk
->wr_max_cred
- csk
->wr_cred
))
1763 csk
->wr_una_cred
= csk
->wr_max_cred
- csk
->wr_cred
;
1766 struct sk_buff
*p
= cxgbit_sock_peek_wr(csk
);
1767 const u32 csum
= (__force u32
)p
->csum
;
1770 pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n",
1771 csk
, csk
->tid
, credits
,
1772 csk
->wr_cred
, csk
->wr_una_cred
);
1776 if (unlikely(credits
< csum
)) {
1777 pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n",
1779 credits
, csk
->wr_cred
, csk
->wr_una_cred
,
1781 p
->csum
= (__force __wsum
)(csum
- credits
);
1785 cxgbit_sock_dequeue_wr(csk
);
1790 if (unlikely(cxgbit_credit_err(csk
))) {
1791 cxgbit_queue_rx_skb(csk
, skb
);
1795 if (rpl
->seq_vld
& CPL_FW4_ACK_FLAGS_SEQVAL
) {
1796 if (unlikely(before(snd_una
, csk
->snd_una
))) {
1797 pr_warn("csk 0x%p,%u, snd_una %u/%u.",
1798 csk
, csk
->tid
, snd_una
,
1803 if (csk
->snd_una
!= snd_una
) {
1804 csk
->snd_una
= snd_una
;
1805 dst_confirm(csk
->dst
);
1806 wake_up(&csk
->ack_waitq
);
1810 if (skb_queue_len(&csk
->txq
))
1811 cxgbit_push_tx_frames(csk
);
1817 static void cxgbit_set_tcb_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1819 struct cxgbit_sock
*csk
;
1820 struct cpl_set_tcb_rpl
*rpl
= (struct cpl_set_tcb_rpl
*)skb
->data
;
1821 unsigned int tid
= GET_TID(rpl
);
1822 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1823 struct tid_info
*t
= lldi
->tids
;
1825 csk
= lookup_tid(t
, tid
);
1826 if (unlikely(!csk
)) {
1827 pr_err("can't find connection for tid %u.\n", tid
);
1830 cxgbit_wake_up(&csk
->com
.wr_wait
, __func__
, rpl
->status
);
1833 cxgbit_put_csk(csk
);
1838 static void cxgbit_rx_data(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1840 struct cxgbit_sock
*csk
;
1841 struct cpl_rx_data
*cpl
= cplhdr(skb
);
1842 unsigned int tid
= GET_TID(cpl
);
1843 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1844 struct tid_info
*t
= lldi
->tids
;
1846 csk
= lookup_tid(t
, tid
);
1847 if (unlikely(!csk
)) {
1848 pr_err("can't find conn. for tid %u.\n", tid
);
1852 cxgbit_queue_rx_skb(csk
, skb
);
1859 __cxgbit_process_rx_cpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1861 spin_lock(&csk
->lock
);
1862 if (csk
->lock_owner
) {
1863 __skb_queue_tail(&csk
->backlogq
, skb
);
1864 spin_unlock(&csk
->lock
);
1868 cxgbit_skcb_rx_backlog_fn(skb
)(csk
, skb
);
1869 spin_unlock(&csk
->lock
);
1872 static void cxgbit_process_rx_cpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1874 cxgbit_get_csk(csk
);
1875 __cxgbit_process_rx_cpl(csk
, skb
);
1876 cxgbit_put_csk(csk
);
1879 static void cxgbit_rx_cpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1881 struct cxgbit_sock
*csk
;
1882 struct cpl_tx_data
*cpl
= cplhdr(skb
);
1883 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1884 struct tid_info
*t
= lldi
->tids
;
1885 unsigned int tid
= GET_TID(cpl
);
1886 u8 opcode
= cxgbit_skcb_rx_opcode(skb
);
1891 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_fw4_ack
;
1894 case CPL_PEER_CLOSE
:
1895 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_peer_close
;
1897 case CPL_CLOSE_CON_RPL
:
1898 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_close_con_rpl
;
1900 case CPL_ABORT_REQ_RSS
:
1901 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_abort_req_rss
;
1903 case CPL_ABORT_RPL_RSS
:
1904 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_abort_rpl_rss
;
1910 csk
= lookup_tid(t
, tid
);
1911 if (unlikely(!csk
)) {
1912 pr_err("can't find conn. for tid %u.\n", tid
);
1917 cxgbit_process_rx_cpl(csk
, skb
);
1919 __cxgbit_process_rx_cpl(csk
, skb
);
1926 cxgbit_cplhandler_func cxgbit_cplhandlers
[NUM_CPL_CMDS
] = {
1927 [CPL_PASS_OPEN_RPL
] = cxgbit_pass_open_rpl
,
1928 [CPL_CLOSE_LISTSRV_RPL
] = cxgbit_close_listsrv_rpl
,
1929 [CPL_PASS_ACCEPT_REQ
] = cxgbit_pass_accept_req
,
1930 [CPL_PASS_ESTABLISH
] = cxgbit_pass_establish
,
1931 [CPL_SET_TCB_RPL
] = cxgbit_set_tcb_rpl
,
1932 [CPL_RX_DATA
] = cxgbit_rx_data
,
1933 [CPL_FW4_ACK
] = cxgbit_rx_cpl
,
1934 [CPL_PEER_CLOSE
] = cxgbit_rx_cpl
,
1935 [CPL_CLOSE_CON_RPL
] = cxgbit_rx_cpl
,
1936 [CPL_ABORT_REQ_RSS
] = cxgbit_rx_cpl
,
1937 [CPL_ABORT_RPL_RSS
] = cxgbit_rx_cpl
,