2 * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver.
4 * Copyright (c) 2010-2015 Chelsio Communications, Inc.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
10 * Written by: Karen Xie (kxie@chelsio.com)
11 * Written by: Rakesh Ranjan (rranjan@chelsio.com)
14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
16 #include <linux/skbuff.h>
17 #include <linux/crypto.h>
18 #include <linux/scatterlist.h>
19 #include <linux/pci.h>
20 #include <scsi/scsi.h>
21 #include <scsi/scsi_cmnd.h>
22 #include <scsi/scsi_host.h>
23 #include <linux/if_vlan.h>
24 #include <linux/inet.h>
26 #include <net/route.h>
28 #include <net/ip6_route.h>
29 #include <net/addrconf.h>
31 #include <linux/inetdevice.h> /* ip_dev_find */
32 #include <linux/module.h>
35 static unsigned int dbg_level
;
39 #define DRV_MODULE_NAME "libcxgbi"
40 #define DRV_MODULE_DESC "Chelsio iSCSI driver library"
41 #define DRV_MODULE_VERSION "0.9.1-ko"
42 #define DRV_MODULE_RELDATE "Apr. 2015"
44 static char version
[] =
45 DRV_MODULE_DESC
" " DRV_MODULE_NAME
46 " v" DRV_MODULE_VERSION
" (" DRV_MODULE_RELDATE
")\n";
48 MODULE_AUTHOR("Chelsio Communications, Inc.");
49 MODULE_DESCRIPTION(DRV_MODULE_DESC
);
50 MODULE_VERSION(DRV_MODULE_VERSION
);
51 MODULE_LICENSE("GPL");
53 module_param(dbg_level
, uint
, 0644);
54 MODULE_PARM_DESC(dbg_level
, "libiscsi debug level (default=0)");
58 * cxgbi device management
59 * maintains a list of the cxgbi devices
61 static LIST_HEAD(cdev_list
);
62 static DEFINE_MUTEX(cdev_mutex
);
64 static LIST_HEAD(cdev_rcu_list
);
65 static DEFINE_SPINLOCK(cdev_rcu_lock
);
67 static inline void cxgbi_decode_sw_tag(u32 sw_tag
, int *idx
, int *age
)
70 *age
= sw_tag
& 0x7FFF;
72 *idx
= (sw_tag
>> 16) & 0x7FFF;
75 int cxgbi_device_portmap_create(struct cxgbi_device
*cdev
, unsigned int base
,
76 unsigned int max_conn
)
78 struct cxgbi_ports_map
*pmap
= &cdev
->pmap
;
80 pmap
->port_csk
= cxgbi_alloc_big_mem(max_conn
*
81 sizeof(struct cxgbi_sock
*),
83 if (!pmap
->port_csk
) {
84 pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev
, max_conn
);
88 pmap
->max_connect
= max_conn
;
89 pmap
->sport_base
= base
;
90 spin_lock_init(&pmap
->lock
);
93 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create
);
95 void cxgbi_device_portmap_cleanup(struct cxgbi_device
*cdev
)
97 struct cxgbi_ports_map
*pmap
= &cdev
->pmap
;
98 struct cxgbi_sock
*csk
;
101 for (i
= 0; i
< pmap
->max_connect
; i
++) {
102 if (pmap
->port_csk
[i
]) {
103 csk
= pmap
->port_csk
[i
];
104 pmap
->port_csk
[i
] = NULL
;
105 log_debug(1 << CXGBI_DBG_SOCK
,
106 "csk 0x%p, cdev 0x%p, offload down.\n",
108 spin_lock_bh(&csk
->lock
);
109 cxgbi_sock_set_flag(csk
, CTPF_OFFLOAD_DOWN
);
110 cxgbi_sock_closed(csk
);
111 spin_unlock_bh(&csk
->lock
);
116 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup
);
118 static inline void cxgbi_device_destroy(struct cxgbi_device
*cdev
)
120 log_debug(1 << CXGBI_DBG_DEV
,
121 "cdev 0x%p, p# %u.\n", cdev
, cdev
->nports
);
122 cxgbi_hbas_remove(cdev
);
123 cxgbi_device_portmap_cleanup(cdev
);
124 cxgbi_ppm_release(cdev
->cdev2ppm(cdev
));
125 if (cdev
->pmap
.max_connect
)
126 cxgbi_free_big_mem(cdev
->pmap
.port_csk
);
130 struct cxgbi_device
*cxgbi_device_register(unsigned int extra
,
133 struct cxgbi_device
*cdev
;
135 cdev
= kzalloc(sizeof(*cdev
) + extra
+ nports
*
136 (sizeof(struct cxgbi_hba
*) +
137 sizeof(struct net_device
*)),
140 pr_warn("nport %d, OOM.\n", nports
);
143 cdev
->ports
= (struct net_device
**)(cdev
+ 1);
144 cdev
->hbas
= (struct cxgbi_hba
**)(((char*)cdev
->ports
) + nports
*
145 sizeof(struct net_device
*));
147 cdev
->dd_data
= ((char *)cdev
->hbas
) +
148 nports
* sizeof(struct cxgbi_hba
*);
149 spin_lock_init(&cdev
->pmap
.lock
);
151 mutex_lock(&cdev_mutex
);
152 list_add_tail(&cdev
->list_head
, &cdev_list
);
153 mutex_unlock(&cdev_mutex
);
155 spin_lock(&cdev_rcu_lock
);
156 list_add_tail_rcu(&cdev
->rcu_node
, &cdev_rcu_list
);
157 spin_unlock(&cdev_rcu_lock
);
159 log_debug(1 << CXGBI_DBG_DEV
,
160 "cdev 0x%p, p# %u.\n", cdev
, nports
);
163 EXPORT_SYMBOL_GPL(cxgbi_device_register
);
165 void cxgbi_device_unregister(struct cxgbi_device
*cdev
)
167 log_debug(1 << CXGBI_DBG_DEV
,
168 "cdev 0x%p, p# %u,%s.\n",
169 cdev
, cdev
->nports
, cdev
->nports
? cdev
->ports
[0]->name
: "");
171 mutex_lock(&cdev_mutex
);
172 list_del(&cdev
->list_head
);
173 mutex_unlock(&cdev_mutex
);
175 spin_lock(&cdev_rcu_lock
);
176 list_del_rcu(&cdev
->rcu_node
);
177 spin_unlock(&cdev_rcu_lock
);
180 cxgbi_device_destroy(cdev
);
182 EXPORT_SYMBOL_GPL(cxgbi_device_unregister
);
184 void cxgbi_device_unregister_all(unsigned int flag
)
186 struct cxgbi_device
*cdev
, *tmp
;
188 mutex_lock(&cdev_mutex
);
189 list_for_each_entry_safe(cdev
, tmp
, &cdev_list
, list_head
) {
190 if ((cdev
->flags
& flag
) == flag
) {
191 mutex_unlock(&cdev_mutex
);
192 cxgbi_device_unregister(cdev
);
193 mutex_lock(&cdev_mutex
);
196 mutex_unlock(&cdev_mutex
);
198 EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all
);
200 struct cxgbi_device
*cxgbi_device_find_by_lldev(void *lldev
)
202 struct cxgbi_device
*cdev
, *tmp
;
204 mutex_lock(&cdev_mutex
);
205 list_for_each_entry_safe(cdev
, tmp
, &cdev_list
, list_head
) {
206 if (cdev
->lldev
== lldev
) {
207 mutex_unlock(&cdev_mutex
);
211 mutex_unlock(&cdev_mutex
);
213 log_debug(1 << CXGBI_DBG_DEV
,
214 "lldev 0x%p, NO match found.\n", lldev
);
217 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev
);
219 struct cxgbi_device
*cxgbi_device_find_by_netdev(struct net_device
*ndev
,
222 struct net_device
*vdev
= NULL
;
223 struct cxgbi_device
*cdev
, *tmp
;
226 if (is_vlan_dev(ndev
)) {
228 ndev
= vlan_dev_real_dev(ndev
);
229 log_debug(1 << CXGBI_DBG_DEV
,
230 "vlan dev %s -> %s.\n", vdev
->name
, ndev
->name
);
233 mutex_lock(&cdev_mutex
);
234 list_for_each_entry_safe(cdev
, tmp
, &cdev_list
, list_head
) {
235 for (i
= 0; i
< cdev
->nports
; i
++) {
236 if (ndev
== cdev
->ports
[i
]) {
237 cdev
->hbas
[i
]->vdev
= vdev
;
238 mutex_unlock(&cdev_mutex
);
245 mutex_unlock(&cdev_mutex
);
246 log_debug(1 << CXGBI_DBG_DEV
,
247 "ndev 0x%p, %s, NO match found.\n", ndev
, ndev
->name
);
250 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev
);
252 struct cxgbi_device
*cxgbi_device_find_by_netdev_rcu(struct net_device
*ndev
,
255 struct net_device
*vdev
= NULL
;
256 struct cxgbi_device
*cdev
;
259 if (is_vlan_dev(ndev
)) {
261 ndev
= vlan_dev_real_dev(ndev
);
262 pr_info("vlan dev %s -> %s.\n", vdev
->name
, ndev
->name
);
266 list_for_each_entry_rcu(cdev
, &cdev_rcu_list
, rcu_node
) {
267 for (i
= 0; i
< cdev
->nports
; i
++) {
268 if (ndev
== cdev
->ports
[i
]) {
269 cdev
->hbas
[i
]->vdev
= vdev
;
279 log_debug(1 << CXGBI_DBG_DEV
,
280 "ndev 0x%p, %s, NO match found.\n", ndev
, ndev
->name
);
283 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu
);
285 static struct cxgbi_device
*cxgbi_device_find_by_mac(struct net_device
*ndev
,
288 struct net_device
*vdev
= NULL
;
289 struct cxgbi_device
*cdev
, *tmp
;
292 if (is_vlan_dev(ndev
)) {
294 ndev
= vlan_dev_real_dev(ndev
);
295 pr_info("vlan dev %s -> %s.\n", vdev
->name
, ndev
->name
);
298 mutex_lock(&cdev_mutex
);
299 list_for_each_entry_safe(cdev
, tmp
, &cdev_list
, list_head
) {
300 for (i
= 0; i
< cdev
->nports
; i
++) {
301 if (!memcmp(ndev
->dev_addr
, cdev
->ports
[i
]->dev_addr
,
303 cdev
->hbas
[i
]->vdev
= vdev
;
304 mutex_unlock(&cdev_mutex
);
311 mutex_unlock(&cdev_mutex
);
312 log_debug(1 << CXGBI_DBG_DEV
,
313 "ndev 0x%p, %s, NO match mac found.\n",
318 void cxgbi_hbas_remove(struct cxgbi_device
*cdev
)
321 struct cxgbi_hba
*chba
;
323 log_debug(1 << CXGBI_DBG_DEV
,
324 "cdev 0x%p, p#%u.\n", cdev
, cdev
->nports
);
326 for (i
= 0; i
< cdev
->nports
; i
++) {
327 chba
= cdev
->hbas
[i
];
329 cdev
->hbas
[i
] = NULL
;
330 iscsi_host_remove(chba
->shost
);
331 pci_dev_put(cdev
->pdev
);
332 iscsi_host_free(chba
->shost
);
336 EXPORT_SYMBOL_GPL(cxgbi_hbas_remove
);
338 int cxgbi_hbas_add(struct cxgbi_device
*cdev
, u64 max_lun
,
339 unsigned int max_id
, struct scsi_host_template
*sht
,
340 struct scsi_transport_template
*stt
)
342 struct cxgbi_hba
*chba
;
343 struct Scsi_Host
*shost
;
346 log_debug(1 << CXGBI_DBG_DEV
, "cdev 0x%p, p#%u.\n", cdev
, cdev
->nports
);
348 for (i
= 0; i
< cdev
->nports
; i
++) {
349 shost
= iscsi_host_alloc(sht
, sizeof(*chba
), 1);
351 pr_info("0x%p, p%d, %s, host alloc failed.\n",
352 cdev
, i
, cdev
->ports
[i
]->name
);
357 shost
->transportt
= stt
;
358 shost
->max_lun
= max_lun
;
359 shost
->max_id
= max_id
;
360 shost
->max_channel
= 0;
361 shost
->max_cmd_len
= 16;
363 chba
= iscsi_host_priv(shost
);
365 chba
->ndev
= cdev
->ports
[i
];
368 log_debug(1 << CXGBI_DBG_DEV
,
369 "cdev 0x%p, p#%d %s: chba 0x%p.\n",
370 cdev
, i
, cdev
->ports
[i
]->name
, chba
);
372 pci_dev_get(cdev
->pdev
);
373 err
= iscsi_host_add(shost
, &cdev
->pdev
->dev
);
375 pr_info("cdev 0x%p, p#%d %s, host add failed.\n",
376 cdev
, i
, cdev
->ports
[i
]->name
);
377 pci_dev_put(cdev
->pdev
);
378 scsi_host_put(shost
);
382 cdev
->hbas
[i
] = chba
;
388 cxgbi_hbas_remove(cdev
);
391 EXPORT_SYMBOL_GPL(cxgbi_hbas_add
);
396 * - source port management
397 * To find a free source port in the port allocation map we use a very simple
398 * rotor scheme to look for the next free port.
400 * If a source port has been specified make sure that it doesn't collide with
401 * our normal source port allocation map. If it's outside the range of our
402 * allocation/deallocation scheme just let them use it.
404 * If the source port is outside our allocation range, the caller is
405 * responsible for keeping track of their port usage.
408 static struct cxgbi_sock
*find_sock_on_port(struct cxgbi_device
*cdev
,
409 unsigned char port_id
)
411 struct cxgbi_ports_map
*pmap
= &cdev
->pmap
;
415 if (!pmap
->max_connect
|| !pmap
->used
)
418 spin_lock_bh(&pmap
->lock
);
420 for (i
= 0; used
&& i
< pmap
->max_connect
; i
++) {
421 struct cxgbi_sock
*csk
= pmap
->port_csk
[i
];
424 if (csk
->port_id
== port_id
) {
425 spin_unlock_bh(&pmap
->lock
);
431 spin_unlock_bh(&pmap
->lock
);
436 static int sock_get_port(struct cxgbi_sock
*csk
)
438 struct cxgbi_device
*cdev
= csk
->cdev
;
439 struct cxgbi_ports_map
*pmap
= &cdev
->pmap
;
444 if (!pmap
->max_connect
) {
445 pr_err("cdev 0x%p, p#%u %s, NO port map.\n",
446 cdev
, csk
->port_id
, cdev
->ports
[csk
->port_id
]->name
);
447 return -EADDRNOTAVAIL
;
450 if (csk
->csk_family
== AF_INET
)
451 port
= &csk
->saddr
.sin_port
;
453 port
= &csk
->saddr6
.sin6_port
;
456 pr_err("source port NON-ZERO %u.\n",
461 spin_lock_bh(&pmap
->lock
);
462 if (pmap
->used
>= pmap
->max_connect
) {
463 spin_unlock_bh(&pmap
->lock
);
464 pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n",
465 cdev
, csk
->port_id
, cdev
->ports
[csk
->port_id
]->name
);
466 return -EADDRNOTAVAIL
;
469 start
= idx
= pmap
->next
;
471 if (++idx
>= pmap
->max_connect
)
473 if (!pmap
->port_csk
[idx
]) {
475 *port
= htons(pmap
->sport_base
+ idx
);
477 pmap
->port_csk
[idx
] = csk
;
478 spin_unlock_bh(&pmap
->lock
);
480 log_debug(1 << CXGBI_DBG_SOCK
,
481 "cdev 0x%p, p#%u %s, p %u, %u.\n",
483 cdev
->ports
[csk
->port_id
]->name
,
484 pmap
->sport_base
+ idx
, pmap
->next
);
487 } while (idx
!= start
);
488 spin_unlock_bh(&pmap
->lock
);
490 /* should not happen */
491 pr_warn("cdev 0x%p, p#%u %s, next %u?\n",
492 cdev
, csk
->port_id
, cdev
->ports
[csk
->port_id
]->name
,
494 return -EADDRNOTAVAIL
;
497 static void sock_put_port(struct cxgbi_sock
*csk
)
499 struct cxgbi_device
*cdev
= csk
->cdev
;
500 struct cxgbi_ports_map
*pmap
= &cdev
->pmap
;
503 if (csk
->csk_family
== AF_INET
)
504 port
= &csk
->saddr
.sin_port
;
506 port
= &csk
->saddr6
.sin6_port
;
509 int idx
= ntohs(*port
) - pmap
->sport_base
;
512 if (idx
< 0 || idx
>= pmap
->max_connect
) {
513 pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n",
515 cdev
->ports
[csk
->port_id
]->name
,
520 spin_lock_bh(&pmap
->lock
);
521 pmap
->port_csk
[idx
] = NULL
;
523 spin_unlock_bh(&pmap
->lock
);
525 log_debug(1 << CXGBI_DBG_SOCK
,
526 "cdev 0x%p, p#%u %s, release %u.\n",
527 cdev
, csk
->port_id
, cdev
->ports
[csk
->port_id
]->name
,
528 pmap
->sport_base
+ idx
);
535 * iscsi tcp connection
537 void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock
*csk
)
539 if (csk
->cpl_close
) {
540 kfree_skb(csk
->cpl_close
);
541 csk
->cpl_close
= NULL
;
543 if (csk
->cpl_abort_req
) {
544 kfree_skb(csk
->cpl_abort_req
);
545 csk
->cpl_abort_req
= NULL
;
547 if (csk
->cpl_abort_rpl
) {
548 kfree_skb(csk
->cpl_abort_rpl
);
549 csk
->cpl_abort_rpl
= NULL
;
552 EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs
);
554 static struct cxgbi_sock
*cxgbi_sock_create(struct cxgbi_device
*cdev
)
556 struct cxgbi_sock
*csk
= kzalloc(sizeof(*csk
), GFP_NOIO
);
559 pr_info("alloc csk %zu failed.\n", sizeof(*csk
));
563 if (cdev
->csk_alloc_cpls(csk
) < 0) {
564 pr_info("csk 0x%p, alloc cpls failed.\n", csk
);
569 spin_lock_init(&csk
->lock
);
570 kref_init(&csk
->refcnt
);
571 skb_queue_head_init(&csk
->receive_queue
);
572 skb_queue_head_init(&csk
->write_queue
);
573 timer_setup(&csk
->retry_timer
, NULL
, 0);
574 init_completion(&csk
->cmpl
);
575 rwlock_init(&csk
->callback_lock
);
578 cxgbi_sock_set_state(csk
, CTP_CLOSED
);
580 log_debug(1 << CXGBI_DBG_SOCK
, "cdev 0x%p, new csk 0x%p.\n", cdev
, csk
);
585 static struct rtable
*find_route_ipv4(struct flowi4
*fl4
,
586 __be32 saddr
, __be32 daddr
,
587 __be16 sport
, __be16 dport
, u8 tos
,
592 rt
= ip_route_output_ports(&init_net
, fl4
, NULL
, daddr
, saddr
,
593 dport
, sport
, IPPROTO_TCP
, tos
, ifindex
);
600 static struct cxgbi_sock
*
601 cxgbi_check_route(struct sockaddr
*dst_addr
, int ifindex
)
603 struct sockaddr_in
*daddr
= (struct sockaddr_in
*)dst_addr
;
604 struct dst_entry
*dst
;
605 struct net_device
*ndev
;
606 struct cxgbi_device
*cdev
;
607 struct rtable
*rt
= NULL
;
610 struct cxgbi_sock
*csk
= NULL
;
611 unsigned int mtu
= 0;
615 rt
= find_route_ipv4(&fl4
, 0, daddr
->sin_addr
.s_addr
, 0,
616 daddr
->sin_port
, 0, ifindex
);
618 pr_info("no route to ipv4 0x%x, port %u.\n",
619 be32_to_cpu(daddr
->sin_addr
.s_addr
),
620 be16_to_cpu(daddr
->sin_port
));
625 n
= dst_neigh_lookup(dst
, &daddr
->sin_addr
.s_addr
);
632 if (rt
->rt_flags
& (RTCF_MULTICAST
| RTCF_BROADCAST
)) {
633 pr_info("multi-cast route %pI4, port %u, dev %s.\n",
634 &daddr
->sin_addr
.s_addr
, ntohs(daddr
->sin_port
),
640 if (ndev
->flags
& IFF_LOOPBACK
) {
641 ndev
= ip_dev_find(&init_net
, daddr
->sin_addr
.s_addr
);
643 pr_info("rt dev %s, loopback -> %s, mtu %u.\n",
644 n
->dev
->name
, ndev
->name
, mtu
);
647 if (!(ndev
->flags
& IFF_UP
) || !netif_carrier_ok(ndev
)) {
648 pr_info("%s interface not up.\n", ndev
->name
);
653 cdev
= cxgbi_device_find_by_netdev(ndev
, &port
);
655 cdev
= cxgbi_device_find_by_mac(ndev
, &port
);
657 pr_info("dst %pI4, %s, NOT cxgbi device.\n",
658 &daddr
->sin_addr
.s_addr
, ndev
->name
);
662 log_debug(1 << CXGBI_DBG_SOCK
,
663 "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n",
664 &daddr
->sin_addr
.s_addr
, ntohs(daddr
->sin_port
),
665 port
, ndev
->name
, cdev
);
667 csk
= cxgbi_sock_create(cdev
);
677 csk
->csk_family
= AF_INET
;
678 csk
->daddr
.sin_addr
.s_addr
= daddr
->sin_addr
.s_addr
;
679 csk
->daddr
.sin_port
= daddr
->sin_port
;
680 csk
->daddr
.sin_family
= daddr
->sin_family
;
681 csk
->saddr
.sin_family
= daddr
->sin_family
;
682 csk
->saddr
.sin_addr
.s_addr
= fl4
.saddr
;
696 #if IS_ENABLED(CONFIG_IPV6)
697 static struct rt6_info
*find_route_ipv6(const struct in6_addr
*saddr
,
698 const struct in6_addr
*daddr
,
703 memset(&fl
, 0, sizeof(fl
));
704 fl
.flowi6_oif
= ifindex
;
706 memcpy(&fl
.saddr
, saddr
, sizeof(struct in6_addr
));
708 memcpy(&fl
.daddr
, daddr
, sizeof(struct in6_addr
));
709 return (struct rt6_info
*)ip6_route_output(&init_net
, NULL
, &fl
);
712 static struct cxgbi_sock
*
713 cxgbi_check_route6(struct sockaddr
*dst_addr
, int ifindex
)
715 struct sockaddr_in6
*daddr6
= (struct sockaddr_in6
*)dst_addr
;
716 struct dst_entry
*dst
;
717 struct net_device
*ndev
;
718 struct cxgbi_device
*cdev
;
719 struct rt6_info
*rt
= NULL
;
721 struct in6_addr pref_saddr
;
722 struct cxgbi_sock
*csk
= NULL
;
723 unsigned int mtu
= 0;
727 rt
= find_route_ipv6(NULL
, &daddr6
->sin6_addr
, ifindex
);
730 pr_info("no route to ipv6 %pI6 port %u\n",
731 daddr6
->sin6_addr
.s6_addr
,
732 be16_to_cpu(daddr6
->sin6_port
));
739 n
= dst_neigh_lookup(dst
, &daddr6
->sin6_addr
);
742 pr_info("%pI6, port %u, dst no neighbour.\n",
743 daddr6
->sin6_addr
.s6_addr
,
744 be16_to_cpu(daddr6
->sin6_port
));
750 if (!(ndev
->flags
& IFF_UP
) || !netif_carrier_ok(ndev
)) {
751 pr_info("%s interface not up.\n", ndev
->name
);
756 if (ipv6_addr_is_multicast(&daddr6
->sin6_addr
)) {
757 pr_info("multi-cast route %pI6 port %u, dev %s.\n",
758 daddr6
->sin6_addr
.s6_addr
,
759 ntohs(daddr6
->sin6_port
), ndev
->name
);
764 cdev
= cxgbi_device_find_by_netdev(ndev
, &port
);
766 cdev
= cxgbi_device_find_by_mac(ndev
, &port
);
768 pr_info("dst %pI6 %s, NOT cxgbi device.\n",
769 daddr6
->sin6_addr
.s6_addr
, ndev
->name
);
773 log_debug(1 << CXGBI_DBG_SOCK
,
774 "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n",
775 daddr6
->sin6_addr
.s6_addr
, ntohs(daddr6
->sin6_port
), port
,
778 csk
= cxgbi_sock_create(cdev
);
788 rt6_get_prefsrc(rt
, &pref_saddr
);
789 if (ipv6_addr_any(&pref_saddr
)) {
790 struct inet6_dev
*idev
= ip6_dst_idev((struct dst_entry
*)rt
);
792 err
= ipv6_dev_get_saddr(&init_net
, idev
? idev
->dev
: NULL
,
793 &daddr6
->sin6_addr
, 0, &pref_saddr
);
795 pr_info("failed to get source address to reach %pI6\n",
801 csk
->csk_family
= AF_INET6
;
802 csk
->daddr6
.sin6_addr
= daddr6
->sin6_addr
;
803 csk
->daddr6
.sin6_port
= daddr6
->sin6_port
;
804 csk
->daddr6
.sin6_family
= daddr6
->sin6_family
;
805 csk
->saddr6
.sin6_family
= daddr6
->sin6_family
;
806 csk
->saddr6
.sin6_addr
= pref_saddr
;
817 cxgbi_sock_closed(csk
);
821 #endif /* IS_ENABLED(CONFIG_IPV6) */
823 void cxgbi_sock_established(struct cxgbi_sock
*csk
, unsigned int snd_isn
,
826 csk
->write_seq
= csk
->snd_nxt
= csk
->snd_una
= snd_isn
;
827 dst_confirm(csk
->dst
);
829 cxgbi_sock_set_state(csk
, CTP_ESTABLISHED
);
831 EXPORT_SYMBOL_GPL(cxgbi_sock_established
);
833 static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock
*csk
)
835 log_debug(1 << CXGBI_DBG_SOCK
,
836 "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n",
837 csk
, csk
->state
, csk
->flags
, csk
->user_data
);
839 if (csk
->state
!= CTP_ESTABLISHED
) {
840 read_lock_bh(&csk
->callback_lock
);
842 iscsi_conn_failure(csk
->user_data
,
843 ISCSI_ERR_TCP_CONN_CLOSE
);
844 read_unlock_bh(&csk
->callback_lock
);
848 void cxgbi_sock_closed(struct cxgbi_sock
*csk
)
850 log_debug(1 << CXGBI_DBG_SOCK
, "csk 0x%p,%u,0x%lx,%u.\n",
851 csk
, (csk
)->state
, (csk
)->flags
, (csk
)->tid
);
852 cxgbi_sock_set_flag(csk
, CTPF_ACTIVE_CLOSE_NEEDED
);
853 if (csk
->state
== CTP_ACTIVE_OPEN
|| csk
->state
== CTP_CLOSED
)
855 if (csk
->saddr
.sin_port
)
858 dst_release(csk
->dst
);
859 csk
->cdev
->csk_release_offload_resources(csk
);
860 cxgbi_sock_set_state(csk
, CTP_CLOSED
);
861 cxgbi_inform_iscsi_conn_closing(csk
);
864 EXPORT_SYMBOL_GPL(cxgbi_sock_closed
);
866 static void need_active_close(struct cxgbi_sock
*csk
)
871 log_debug(1 << CXGBI_DBG_SOCK
, "csk 0x%p,%u,0x%lx,%u.\n",
872 csk
, (csk
)->state
, (csk
)->flags
, (csk
)->tid
);
873 spin_lock_bh(&csk
->lock
);
875 dst_confirm(csk
->dst
);
876 data_lost
= skb_queue_len(&csk
->receive_queue
);
877 __skb_queue_purge(&csk
->receive_queue
);
879 if (csk
->state
== CTP_ACTIVE_OPEN
)
880 cxgbi_sock_set_flag(csk
, CTPF_ACTIVE_CLOSE_NEEDED
);
881 else if (csk
->state
== CTP_ESTABLISHED
) {
883 cxgbi_sock_set_state(csk
, CTP_ACTIVE_CLOSE
);
884 } else if (csk
->state
== CTP_PASSIVE_CLOSE
) {
886 cxgbi_sock_set_state(csk
, CTP_CLOSE_WAIT_2
);
890 if (!cxgbi_sock_flag(csk
, CTPF_LOGOUT_RSP_RCVD
) ||
892 csk
->cdev
->csk_send_abort_req(csk
);
894 csk
->cdev
->csk_send_close_req(csk
);
897 spin_unlock_bh(&csk
->lock
);
900 void cxgbi_sock_fail_act_open(struct cxgbi_sock
*csk
, int errno
)
902 pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n",
903 csk
, csk
->state
, csk
->flags
,
904 &csk
->saddr
.sin_addr
.s_addr
, csk
->saddr
.sin_port
,
905 &csk
->daddr
.sin_addr
.s_addr
, csk
->daddr
.sin_port
,
908 cxgbi_sock_set_state(csk
, CTP_CONNECTING
);
910 cxgbi_sock_closed(csk
);
912 EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open
);
914 void cxgbi_sock_act_open_req_arp_failure(void *handle
, struct sk_buff
*skb
)
916 struct cxgbi_sock
*csk
= (struct cxgbi_sock
*)skb
->sk
;
917 struct module
*owner
= csk
->cdev
->owner
;
919 log_debug(1 << CXGBI_DBG_SOCK
, "csk 0x%p,%u,0x%lx,%u.\n",
920 csk
, (csk
)->state
, (csk
)->flags
, (csk
)->tid
);
922 spin_lock_bh(&csk
->lock
);
923 if (csk
->state
== CTP_ACTIVE_OPEN
)
924 cxgbi_sock_fail_act_open(csk
, -EHOSTUNREACH
);
925 spin_unlock_bh(&csk
->lock
);
931 EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure
);
933 void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock
*csk
)
936 spin_lock_bh(&csk
->lock
);
938 cxgbi_sock_set_flag(csk
, CTPF_ABORT_RPL_RCVD
);
939 if (cxgbi_sock_flag(csk
, CTPF_ABORT_RPL_PENDING
)) {
940 cxgbi_sock_clear_flag(csk
, CTPF_ABORT_RPL_PENDING
);
941 if (cxgbi_sock_flag(csk
, CTPF_ABORT_REQ_RCVD
))
942 pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n",
943 csk
, csk
->state
, csk
->flags
, csk
->tid
);
944 cxgbi_sock_closed(csk
);
947 spin_unlock_bh(&csk
->lock
);
950 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl
);
952 void cxgbi_sock_rcv_peer_close(struct cxgbi_sock
*csk
)
954 log_debug(1 << CXGBI_DBG_SOCK
, "csk 0x%p,%u,0x%lx,%u.\n",
955 csk
, (csk
)->state
, (csk
)->flags
, (csk
)->tid
);
957 spin_lock_bh(&csk
->lock
);
959 if (cxgbi_sock_flag(csk
, CTPF_ABORT_RPL_PENDING
))
962 switch (csk
->state
) {
963 case CTP_ESTABLISHED
:
964 cxgbi_sock_set_state(csk
, CTP_PASSIVE_CLOSE
);
966 case CTP_ACTIVE_CLOSE
:
967 cxgbi_sock_set_state(csk
, CTP_CLOSE_WAIT_2
);
969 case CTP_CLOSE_WAIT_1
:
970 cxgbi_sock_closed(csk
);
975 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
976 csk
, csk
->state
, csk
->flags
, csk
->tid
);
978 cxgbi_inform_iscsi_conn_closing(csk
);
980 spin_unlock_bh(&csk
->lock
);
983 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close
);
985 void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock
*csk
, u32 snd_nxt
)
987 log_debug(1 << CXGBI_DBG_SOCK
, "csk 0x%p,%u,0x%lx,%u.\n",
988 csk
, (csk
)->state
, (csk
)->flags
, (csk
)->tid
);
990 spin_lock_bh(&csk
->lock
);
992 csk
->snd_una
= snd_nxt
- 1;
993 if (cxgbi_sock_flag(csk
, CTPF_ABORT_RPL_PENDING
))
996 switch (csk
->state
) {
997 case CTP_ACTIVE_CLOSE
:
998 cxgbi_sock_set_state(csk
, CTP_CLOSE_WAIT_1
);
1000 case CTP_CLOSE_WAIT_1
:
1001 case CTP_CLOSE_WAIT_2
:
1002 cxgbi_sock_closed(csk
);
1007 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
1008 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1011 spin_unlock_bh(&csk
->lock
);
1012 cxgbi_sock_put(csk
);
1014 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl
);
1016 void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock
*csk
, unsigned int credits
,
1017 unsigned int snd_una
, int seq_chk
)
1019 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1020 "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n",
1021 csk
, csk
->state
, csk
->flags
, csk
->tid
, credits
,
1022 csk
->wr_cred
, csk
->wr_una_cred
, snd_una
, seq_chk
);
1024 spin_lock_bh(&csk
->lock
);
1026 csk
->wr_cred
+= credits
;
1027 if (csk
->wr_una_cred
> csk
->wr_max_cred
- csk
->wr_cred
)
1028 csk
->wr_una_cred
= csk
->wr_max_cred
- csk
->wr_cred
;
1031 struct sk_buff
*p
= cxgbi_sock_peek_wr(csk
);
1034 pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n",
1035 csk
, csk
->state
, csk
->flags
, csk
->tid
, credits
,
1036 csk
->wr_cred
, csk
->wr_una_cred
);
1040 if (unlikely(credits
< p
->csum
)) {
1041 pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n",
1042 csk
, csk
->state
, csk
->flags
, csk
->tid
,
1043 credits
, csk
->wr_cred
, csk
->wr_una_cred
,
1048 cxgbi_sock_dequeue_wr(csk
);
1054 cxgbi_sock_check_wr_invariants(csk
);
1057 if (unlikely(before(snd_una
, csk
->snd_una
))) {
1058 pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.",
1059 csk
, csk
->state
, csk
->flags
, csk
->tid
, snd_una
,
1064 if (csk
->snd_una
!= snd_una
) {
1065 csk
->snd_una
= snd_una
;
1066 dst_confirm(csk
->dst
);
1070 if (skb_queue_len(&csk
->write_queue
)) {
1071 if (csk
->cdev
->csk_push_tx_frames(csk
, 0))
1072 cxgbi_conn_tx_open(csk
);
1074 cxgbi_conn_tx_open(csk
);
1076 spin_unlock_bh(&csk
->lock
);
1078 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack
);
1080 static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock
*csk
,
1085 while (i
< csk
->cdev
->nmtus
- 1 && csk
->cdev
->mtus
[i
+ 1] <= mtu
)
1091 unsigned int cxgbi_sock_select_mss(struct cxgbi_sock
*csk
, unsigned int pmtu
)
1094 struct dst_entry
*dst
= csk
->dst
;
1096 csk
->advmss
= dst_metric_advmss(dst
);
1098 if (csk
->advmss
> pmtu
- 40)
1099 csk
->advmss
= pmtu
- 40;
1100 if (csk
->advmss
< csk
->cdev
->mtus
[0] - 40)
1101 csk
->advmss
= csk
->cdev
->mtus
[0] - 40;
1102 idx
= cxgbi_sock_find_best_mtu(csk
, csk
->advmss
+ 40);
1106 EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss
);
1108 void cxgbi_sock_skb_entail(struct cxgbi_sock
*csk
, struct sk_buff
*skb
)
1110 cxgbi_skcb_tcp_seq(skb
) = csk
->write_seq
;
1111 __skb_queue_tail(&csk
->write_queue
, skb
);
1113 EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail
);
1115 void cxgbi_sock_purge_wr_queue(struct cxgbi_sock
*csk
)
1117 struct sk_buff
*skb
;
1119 while ((skb
= cxgbi_sock_dequeue_wr(csk
)) != NULL
)
1122 EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue
);
1124 void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock
*csk
)
1126 int pending
= cxgbi_sock_count_pending_wrs(csk
);
1128 if (unlikely(csk
->wr_cred
+ pending
!= csk
->wr_max_cred
))
1129 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1130 csk
, csk
->tid
, csk
->wr_cred
, pending
, csk
->wr_max_cred
);
1132 EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants
);
1134 static int cxgbi_sock_send_pdus(struct cxgbi_sock
*csk
, struct sk_buff
*skb
)
1136 struct cxgbi_device
*cdev
= csk
->cdev
;
1137 struct sk_buff
*next
;
1138 int err
, copied
= 0;
1140 spin_lock_bh(&csk
->lock
);
1142 if (csk
->state
!= CTP_ESTABLISHED
) {
1143 log_debug(1 << CXGBI_DBG_PDU_TX
,
1144 "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n",
1145 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1151 log_debug(1 << CXGBI_DBG_PDU_TX
,
1152 "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n",
1153 csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->err
);
1158 if (csk
->write_seq
- csk
->snd_una
>= csk
->snd_win
) {
1159 log_debug(1 << CXGBI_DBG_PDU_TX
,
1160 "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n",
1161 csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->write_seq
,
1162 csk
->snd_una
, csk
->snd_win
);
1168 int frags
= skb_shinfo(skb
)->nr_frags
+
1169 (skb
->len
!= skb
->data_len
);
1171 if (unlikely(skb_headroom(skb
) < cdev
->skb_tx_rsvd
)) {
1172 pr_err("csk 0x%p, skb head %u < %u.\n",
1173 csk
, skb_headroom(skb
), cdev
->skb_tx_rsvd
);
1178 if (frags
>= SKB_WR_LIST_SIZE
) {
1179 pr_err("csk 0x%p, frags %d, %u,%u >%u.\n",
1180 csk
, skb_shinfo(skb
)->nr_frags
, skb
->len
,
1181 skb
->data_len
, (uint
)(SKB_WR_LIST_SIZE
));
1188 cxgbi_skcb_set_flag(skb
, SKCBF_TX_NEED_HDR
);
1189 cxgbi_sock_skb_entail(csk
, skb
);
1191 csk
->write_seq
+= skb
->len
+
1192 cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb
));
1196 if (likely(skb_queue_len(&csk
->write_queue
)))
1197 cdev
->csk_push_tx_frames(csk
, 1);
1199 spin_unlock_bh(&csk
->lock
);
1203 if (copied
== 0 && err
== -EPIPE
)
1204 copied
= csk
->err
? csk
->err
: -EPIPE
;
1211 scmd_get_params(struct scsi_cmnd
*sc
, struct scatterlist
**sgl
,
1212 unsigned int *sgcnt
, unsigned int *dlen
,
1215 struct scsi_data_buffer
*sdb
= prot
? scsi_prot(sc
) : &sc
->sdb
;
1217 *sgl
= sdb
->table
.sgl
;
1218 *sgcnt
= sdb
->table
.nents
;
1219 *dlen
= sdb
->length
;
1220 /* Caution: for protection sdb, sdb->length is invalid */
1223 void cxgbi_ddp_set_one_ppod(struct cxgbi_pagepod
*ppod
,
1224 struct cxgbi_task_tag_info
*ttinfo
,
1225 struct scatterlist
**sg_pp
, unsigned int *sg_off
)
1227 struct scatterlist
*sg
= sg_pp
? *sg_pp
: NULL
;
1228 unsigned int offset
= sg_off
? *sg_off
: 0;
1229 dma_addr_t addr
= 0UL;
1230 unsigned int len
= 0;
1233 memcpy(ppod
, &ttinfo
->hdr
, sizeof(struct cxgbi_pagepod_hdr
));
1236 addr
= sg_dma_address(sg
);
1237 len
= sg_dma_len(sg
);
1240 for (i
= 0; i
< PPOD_PAGES_MAX
; i
++) {
1242 ppod
->addr
[i
] = cpu_to_be64(addr
+ offset
);
1243 offset
+= PAGE_SIZE
;
1244 if (offset
== (len
+ sg
->offset
)) {
1248 addr
= sg_dma_address(sg
);
1249 len
= sg_dma_len(sg
);
1253 ppod
->addr
[i
] = 0ULL;
1258 * the fifth address needs to be repeated in the next ppod, so do
1266 if (offset
== len
) {
1270 addr
= sg_dma_address(sg
);
1271 len
= sg_dma_len(sg
);
1274 ppod
->addr
[i
] = sg
? cpu_to_be64(addr
+ offset
) : 0ULL;
1276 EXPORT_SYMBOL_GPL(cxgbi_ddp_set_one_ppod
);
1279 * APIs interacting with open-iscsi libraries
1282 static unsigned char padding
[4];
1284 void cxgbi_ddp_ppm_setup(void **ppm_pp
, struct cxgbi_device
*cdev
,
1285 struct cxgbi_tag_format
*tformat
, unsigned int ppmax
,
1286 unsigned int llimit
, unsigned int start
,
1287 unsigned int rsvd_factor
)
1289 int err
= cxgbi_ppm_init(ppm_pp
, cdev
->ports
[0], cdev
->pdev
,
1290 cdev
->lldev
, tformat
, ppmax
, llimit
, start
,
1294 struct cxgbi_ppm
*ppm
= (struct cxgbi_ppm
*)(*ppm_pp
);
1296 if (ppm
->ppmax
< 1024 ||
1297 ppm
->tformat
.pgsz_idx_dflt
>= DDP_PGIDX_MAX
)
1298 cdev
->flags
|= CXGBI_FLAG_DDP_OFF
;
1301 cdev
->flags
|= CXGBI_FLAG_DDP_OFF
;
1304 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppm_setup
);
1306 static int cxgbi_ddp_sgl_check(struct scatterlist
*sgl
, int nents
)
1309 int last_sgidx
= nents
- 1;
1310 struct scatterlist
*sg
= sgl
;
1312 for (i
= 0; i
< nents
; i
++, sg
= sg_next(sg
)) {
1313 unsigned int len
= sg
->length
+ sg
->offset
;
1315 if ((sg
->offset
& 0x3) || (i
&& sg
->offset
) ||
1316 ((i
!= last_sgidx
) && len
!= PAGE_SIZE
)) {
1317 log_debug(1 << CXGBI_DBG_DDP
,
1318 "sg %u/%u, %u,%u, not aligned.\n",
1319 i
, nents
, sg
->offset
, sg
->length
);
1328 static int cxgbi_ddp_reserve(struct cxgbi_conn
*cconn
,
1329 struct cxgbi_task_data
*tdata
, u32 sw_tag
,
1330 unsigned int xferlen
)
1332 struct cxgbi_sock
*csk
= cconn
->cep
->csk
;
1333 struct cxgbi_device
*cdev
= csk
->cdev
;
1334 struct cxgbi_ppm
*ppm
= cdev
->cdev2ppm(cdev
);
1335 struct cxgbi_task_tag_info
*ttinfo
= &tdata
->ttinfo
;
1336 struct scatterlist
*sgl
= ttinfo
->sgl
;
1337 unsigned int sgcnt
= ttinfo
->nents
;
1338 unsigned int sg_offset
= sgl
->offset
;
1341 if (cdev
->flags
& CXGBI_FLAG_DDP_OFF
) {
1342 log_debug(1 << CXGBI_DBG_DDP
,
1343 "cdev 0x%p DDP off.\n", cdev
);
1347 if (!ppm
|| xferlen
< DDP_THRESHOLD
|| !sgcnt
||
1348 ppm
->tformat
.pgsz_idx_dflt
>= DDP_PGIDX_MAX
) {
1349 log_debug(1 << CXGBI_DBG_DDP
,
1350 "ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n",
1351 ppm
, ppm
? ppm
->tformat
.pgsz_idx_dflt
: DDP_PGIDX_MAX
,
1352 xferlen
, ttinfo
->nents
);
1356 /* make sure the buffer is suitable for ddp */
1357 if (cxgbi_ddp_sgl_check(sgl
, sgcnt
) < 0)
1360 ttinfo
->nr_pages
= (xferlen
+ sgl
->offset
+ (1 << PAGE_SHIFT
) - 1) >>
1364 * the ddp tag will be used for the itt in the outgoing pdu,
1365 * the itt genrated by libiscsi is saved in the ppm and can be
1366 * retrieved via the ddp tag
1368 err
= cxgbi_ppm_ppods_reserve(ppm
, ttinfo
->nr_pages
, 0, &ttinfo
->idx
,
1369 &ttinfo
->tag
, (unsigned long)sw_tag
);
1374 ttinfo
->npods
= err
;
1376 /* setup dma from scsi command sgl */
1378 err
= dma_map_sg(&ppm
->pdev
->dev
, sgl
, sgcnt
, DMA_FROM_DEVICE
);
1379 sgl
->offset
= sg_offset
;
1381 pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n",
1382 __func__
, sw_tag
, xferlen
, sgcnt
);
1385 if (err
!= ttinfo
->nr_pages
) {
1386 log_debug(1 << CXGBI_DBG_DDP
,
1387 "%s: sw tag 0x%x, xfer %u, sgl %u, dma count %d.\n",
1388 __func__
, sw_tag
, xferlen
, sgcnt
, err
);
1391 ttinfo
->flags
|= CXGBI_PPOD_INFO_FLAG_MAPPED
;
1392 ttinfo
->cid
= csk
->port_id
;
1394 cxgbi_ppm_make_ppod_hdr(ppm
, ttinfo
->tag
, csk
->tid
, sgl
->offset
,
1395 xferlen
, &ttinfo
->hdr
);
1397 if (cdev
->flags
& CXGBI_FLAG_USE_PPOD_OFLDQ
) {
1398 /* write ppod from xmit_pdu (of iscsi_scsi_command pdu) */
1399 ttinfo
->flags
|= CXGBI_PPOD_INFO_FLAG_VALID
;
1401 /* write ppod from control queue now */
1402 err
= cdev
->csk_ddp_set_map(ppm
, csk
, ttinfo
);
1410 cxgbi_ppm_ppod_release(ppm
, ttinfo
->idx
);
1412 if (ttinfo
->flags
& CXGBI_PPOD_INFO_FLAG_MAPPED
) {
1413 ttinfo
->flags
&= ~CXGBI_PPOD_INFO_FLAG_MAPPED
;
1414 dma_unmap_sg(&ppm
->pdev
->dev
, sgl
, sgcnt
, DMA_FROM_DEVICE
);
1419 static void task_release_itt(struct iscsi_task
*task
, itt_t hdr_itt
)
1421 struct scsi_cmnd
*sc
= task
->sc
;
1422 struct iscsi_tcp_conn
*tcp_conn
= task
->conn
->dd_data
;
1423 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
1424 struct cxgbi_device
*cdev
= cconn
->chba
->cdev
;
1425 struct cxgbi_ppm
*ppm
= cdev
->cdev2ppm(cdev
);
1426 u32 tag
= ntohl((__force u32
)hdr_itt
);
1428 log_debug(1 << CXGBI_DBG_DDP
,
1429 "cdev 0x%p, task 0x%p, release tag 0x%x.\n",
1431 if (sc
&& sc
->sc_data_direction
== DMA_FROM_DEVICE
&&
1432 cxgbi_ppm_is_ddp_tag(ppm
, tag
)) {
1433 struct cxgbi_task_data
*tdata
= iscsi_task_cxgbi_data(task
);
1434 struct cxgbi_task_tag_info
*ttinfo
= &tdata
->ttinfo
;
1436 if (!(cdev
->flags
& CXGBI_FLAG_USE_PPOD_OFLDQ
))
1437 cdev
->csk_ddp_clear_map(cdev
, ppm
, ttinfo
);
1438 cxgbi_ppm_ppod_release(ppm
, ttinfo
->idx
);
1439 dma_unmap_sg(&ppm
->pdev
->dev
, ttinfo
->sgl
, ttinfo
->nents
,
1444 static inline u32
cxgbi_build_sw_tag(u32 idx
, u32 age
)
1446 /* assume idx and age both are < 0x7FFF (32767) */
1447 return (idx
<< 16) | age
;
1450 static int task_reserve_itt(struct iscsi_task
*task
, itt_t
*hdr_itt
)
1452 struct scsi_cmnd
*sc
= task
->sc
;
1453 struct iscsi_conn
*conn
= task
->conn
;
1454 struct iscsi_session
*sess
= conn
->session
;
1455 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1456 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
1457 struct cxgbi_device
*cdev
= cconn
->chba
->cdev
;
1458 struct cxgbi_ppm
*ppm
= cdev
->cdev2ppm(cdev
);
1459 u32 sw_tag
= cxgbi_build_sw_tag(task
->itt
, sess
->age
);
1463 if (sc
&& sc
->sc_data_direction
== DMA_FROM_DEVICE
) {
1464 struct cxgbi_task_data
*tdata
= iscsi_task_cxgbi_data(task
);
1465 struct cxgbi_task_tag_info
*ttinfo
= &tdata
->ttinfo
;
1467 scmd_get_params(sc
, &ttinfo
->sgl
, &ttinfo
->nents
,
1469 err
= cxgbi_ddp_reserve(cconn
, tdata
, sw_tag
, tdata
->dlen
);
1473 log_debug(1 << CXGBI_DBG_DDP
,
1474 "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n",
1475 cconn
->cep
->csk
, task
, tdata
->dlen
,
1480 err
= cxgbi_ppm_make_non_ddp_tag(ppm
, sw_tag
, &tag
);
1484 /* the itt need to sent in big-endian order */
1485 *hdr_itt
= (__force itt_t
)htonl(tag
);
1487 log_debug(1 << CXGBI_DBG_DDP
,
1488 "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n",
1489 cdev
, task
, sw_tag
, task
->itt
, sess
->age
, tag
, *hdr_itt
);
1493 void cxgbi_parse_pdu_itt(struct iscsi_conn
*conn
, itt_t itt
, int *idx
, int *age
)
1495 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1496 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
1497 struct cxgbi_device
*cdev
= cconn
->chba
->cdev
;
1498 struct cxgbi_ppm
*ppm
= cdev
->cdev2ppm(cdev
);
1499 u32 tag
= ntohl((__force u32
)itt
);
1503 if (cxgbi_ppm_is_ddp_tag(ppm
, tag
))
1504 sw_bits
= cxgbi_ppm_get_tag_caller_data(ppm
, tag
);
1506 sw_bits
= cxgbi_ppm_decode_non_ddp_tag(ppm
, tag
);
1511 cxgbi_decode_sw_tag(sw_bits
, idx
, age
);
1512 log_debug(1 << CXGBI_DBG_DDP
,
1513 "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n",
1514 cdev
, tag
, itt
, sw_bits
, idx
? *idx
: 0xFFFFF,
1517 EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt
);
1519 void cxgbi_conn_tx_open(struct cxgbi_sock
*csk
)
1521 struct iscsi_conn
*conn
= csk
->user_data
;
1524 log_debug(1 << CXGBI_DBG_SOCK
,
1525 "csk 0x%p, cid %d.\n", csk
, conn
->id
);
1526 iscsi_conn_queue_work(conn
);
1529 EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open
);
1532 * pdu receive, interact with libiscsi_tcp
1534 static inline int read_pdu_skb(struct iscsi_conn
*conn
,
1535 struct sk_buff
*skb
,
1536 unsigned int offset
,
1542 bytes_read
= iscsi_tcp_recv_skb(conn
, skb
, offset
, offloaded
, &status
);
1544 case ISCSI_TCP_CONN_ERR
:
1545 pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n",
1546 skb
, offset
, offloaded
);
1548 case ISCSI_TCP_SUSPENDED
:
1549 log_debug(1 << CXGBI_DBG_PDU_RX
,
1550 "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n",
1551 skb
, offset
, offloaded
, bytes_read
);
1552 /* no transfer - just have caller flush queue */
1554 case ISCSI_TCP_SKB_DONE
:
1555 pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n",
1556 skb
, offset
, offloaded
);
1558 * pdus should always fit in the skb and we should get
1559 * segment done notifcation.
1561 iscsi_conn_printk(KERN_ERR
, conn
, "Invalid pdu or skb.");
1563 case ISCSI_TCP_SEGMENT_DONE
:
1564 log_debug(1 << CXGBI_DBG_PDU_RX
,
1565 "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n",
1566 skb
, offset
, offloaded
, bytes_read
);
1569 pr_info("skb 0x%p, off %u, %d, invalid status %d.\n",
1570 skb
, offset
, offloaded
, status
);
1576 skb_read_pdu_bhs(struct cxgbi_sock
*csk
, struct iscsi_conn
*conn
,
1577 struct sk_buff
*skb
)
1579 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1582 log_debug(1 << CXGBI_DBG_PDU_RX
,
1583 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
1584 conn
, skb
, skb
->len
, cxgbi_skcb_flags(skb
));
1586 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn
)) {
1587 pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn
, skb
);
1588 iscsi_conn_failure(conn
, ISCSI_ERR_PROTO
);
1592 if (conn
->hdrdgst_en
&&
1593 cxgbi_skcb_test_flag(skb
, SKCBF_RX_HCRC_ERR
)) {
1594 pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn
, skb
);
1595 iscsi_conn_failure(conn
, ISCSI_ERR_HDR_DGST
);
1599 if (cxgbi_skcb_test_flag(skb
, SKCBF_RX_ISCSI_COMPL
) &&
1600 cxgbi_skcb_test_flag(skb
, SKCBF_RX_DATA_DDPD
)) {
1601 /* If completion flag is set and data is directly
1602 * placed in to the host memory then update
1603 * task->exp_datasn to the datasn in completion
1604 * iSCSI hdr as T6 adapter generates completion only
1605 * for the last pdu of a sequence.
1607 itt_t itt
= ((struct iscsi_data
*)skb
->data
)->itt
;
1608 struct iscsi_task
*task
= iscsi_itt_to_ctask(conn
, itt
);
1609 u32 data_sn
= be32_to_cpu(((struct iscsi_data
*)
1610 skb
->data
)->datasn
);
1611 if (task
&& task
->sc
) {
1612 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
1614 tcp_task
->exp_datasn
= data_sn
;
1618 err
= read_pdu_skb(conn
, skb
, 0, 0);
1619 if (likely(err
>= 0)) {
1620 struct iscsi_hdr
*hdr
= (struct iscsi_hdr
*)skb
->data
;
1621 u8 opcode
= hdr
->opcode
& ISCSI_OPCODE_MASK
;
1623 if (unlikely(opcode
== ISCSI_OP_LOGOUT_RSP
))
1624 cxgbi_sock_set_flag(csk
, CTPF_LOGOUT_RSP_RCVD
);
1630 static int skb_read_pdu_data(struct iscsi_conn
*conn
, struct sk_buff
*lskb
,
1631 struct sk_buff
*skb
, unsigned int offset
)
1633 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1635 int opcode
= tcp_conn
->in
.hdr
->opcode
& ISCSI_OPCODE_MASK
;
1637 log_debug(1 << CXGBI_DBG_PDU_RX
,
1638 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
1639 conn
, skb
, skb
->len
, cxgbi_skcb_flags(skb
));
1641 if (conn
->datadgst_en
&&
1642 cxgbi_skcb_test_flag(lskb
, SKCBF_RX_DCRC_ERR
)) {
1643 pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n",
1644 conn
, lskb
, cxgbi_skcb_flags(lskb
));
1645 iscsi_conn_failure(conn
, ISCSI_ERR_DATA_DGST
);
1649 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn
))
1652 /* coalesced, add header digest length */
1653 if (lskb
== skb
&& conn
->hdrdgst_en
)
1654 offset
+= ISCSI_DIGEST_SIZE
;
1656 if (cxgbi_skcb_test_flag(lskb
, SKCBF_RX_DATA_DDPD
))
1659 if (opcode
== ISCSI_OP_SCSI_DATA_IN
)
1660 log_debug(1 << CXGBI_DBG_PDU_RX
,
1661 "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n",
1662 skb
, opcode
, ntohl(tcp_conn
->in
.hdr
->itt
),
1663 tcp_conn
->in
.datalen
, offloaded
? "is" : "not");
1665 return read_pdu_skb(conn
, skb
, offset
, offloaded
);
1668 static void csk_return_rx_credits(struct cxgbi_sock
*csk
, int copied
)
1670 struct cxgbi_device
*cdev
= csk
->cdev
;
1674 log_debug(1 << CXGBI_DBG_PDU_RX
,
1675 "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n",
1676 csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->copied_seq
,
1677 csk
->rcv_wup
, cdev
->rx_credit_thres
,
1680 if (!cdev
->rx_credit_thres
)
1683 if (csk
->state
!= CTP_ESTABLISHED
)
1686 credits
= csk
->copied_seq
- csk
->rcv_wup
;
1687 if (unlikely(!credits
))
1689 must_send
= credits
+ 16384 >= csk
->rcv_win
;
1690 if (must_send
|| credits
>= cdev
->rx_credit_thres
)
1691 csk
->rcv_wup
+= cdev
->csk_send_rx_credits(csk
, credits
);
1694 void cxgbi_conn_pdu_ready(struct cxgbi_sock
*csk
)
1696 struct cxgbi_device
*cdev
= csk
->cdev
;
1697 struct iscsi_conn
*conn
= csk
->user_data
;
1698 struct sk_buff
*skb
;
1699 unsigned int read
= 0;
1702 log_debug(1 << CXGBI_DBG_PDU_RX
,
1703 "csk 0x%p, conn 0x%p.\n", csk
, conn
);
1705 if (unlikely(!conn
|| conn
->suspend_rx
)) {
1706 log_debug(1 << CXGBI_DBG_PDU_RX
,
1707 "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n",
1708 csk
, conn
, conn
? conn
->id
: 0xFF,
1709 conn
? conn
->suspend_rx
: 0xFF);
1714 skb
= skb_peek(&csk
->receive_queue
);
1716 !(cxgbi_skcb_test_flag(skb
, SKCBF_RX_STATUS
))) {
1718 log_debug(1 << CXGBI_DBG_PDU_RX
,
1719 "skb 0x%p, NOT ready 0x%lx.\n",
1720 skb
, cxgbi_skcb_flags(skb
));
1723 __skb_unlink(skb
, &csk
->receive_queue
);
1725 read
+= cxgbi_skcb_rx_pdulen(skb
);
1726 log_debug(1 << CXGBI_DBG_PDU_RX
,
1727 "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n",
1728 csk
, skb
, skb
->len
, cxgbi_skcb_flags(skb
),
1729 cxgbi_skcb_rx_pdulen(skb
));
1731 if (cxgbi_skcb_test_flag(skb
, SKCBF_RX_COALESCED
)) {
1732 err
= skb_read_pdu_bhs(csk
, conn
, skb
);
1734 pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, "
1735 "f 0x%lx, plen %u.\n",
1737 cxgbi_skcb_flags(skb
),
1738 cxgbi_skcb_rx_pdulen(skb
));
1741 err
= skb_read_pdu_data(conn
, skb
, skb
,
1742 err
+ cdev
->skb_rx_extra
);
1744 pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, "
1745 "f 0x%lx, plen %u.\n",
1747 cxgbi_skcb_flags(skb
),
1748 cxgbi_skcb_rx_pdulen(skb
));
1750 err
= skb_read_pdu_bhs(csk
, conn
, skb
);
1752 pr_err("bhs, csk 0x%p, skb 0x%p,%u, "
1753 "f 0x%lx, plen %u.\n",
1755 cxgbi_skcb_flags(skb
),
1756 cxgbi_skcb_rx_pdulen(skb
));
1760 if (cxgbi_skcb_test_flag(skb
, SKCBF_RX_DATA
)) {
1761 struct sk_buff
*dskb
;
1763 dskb
= skb_peek(&csk
->receive_queue
);
1765 pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx,"
1766 " plen %u, NO data.\n",
1768 cxgbi_skcb_flags(skb
),
1769 cxgbi_skcb_rx_pdulen(skb
));
1773 __skb_unlink(dskb
, &csk
->receive_queue
);
1775 err
= skb_read_pdu_data(conn
, skb
, dskb
, 0);
1777 pr_err("data, csk 0x%p, skb 0x%p,%u, "
1778 "f 0x%lx, plen %u, dskb 0x%p,"
1781 cxgbi_skcb_flags(skb
),
1782 cxgbi_skcb_rx_pdulen(skb
),
1786 err
= skb_read_pdu_data(conn
, skb
, skb
, 0);
1795 log_debug(1 << CXGBI_DBG_PDU_RX
, "csk 0x%p, read %u.\n", csk
, read
);
1797 csk
->copied_seq
+= read
;
1798 csk_return_rx_credits(csk
, read
);
1799 conn
->rxdata_octets
+= read
;
1803 pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n",
1804 csk
, conn
, err
, read
);
1805 iscsi_conn_failure(conn
, ISCSI_ERR_CONN_FAILED
);
1808 EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready
);
1810 static int sgl_seek_offset(struct scatterlist
*sgl
, unsigned int sgcnt
,
1811 unsigned int offset
, unsigned int *off
,
1812 struct scatterlist
**sgp
)
1815 struct scatterlist
*sg
;
1817 for_each_sg(sgl
, sg
, sgcnt
, i
) {
1818 if (offset
< sg
->length
) {
1823 offset
-= sg
->length
;
1828 static int sgl_read_to_frags(struct scatterlist
*sg
, unsigned int sgoffset
,
1829 unsigned int dlen
, struct page_frag
*frags
,
1832 unsigned int datalen
= dlen
;
1833 unsigned int sglen
= sg
->length
- sgoffset
;
1834 struct page
*page
= sg_page(sg
);
1844 pr_warn("sg %d NULL, len %u/%u.\n",
1853 copy
= min(datalen
, sglen
);
1854 if (i
&& page
== frags
[i
- 1].page
&&
1855 sgoffset
+ sg
->offset
==
1856 frags
[i
- 1].offset
+ frags
[i
- 1].size
) {
1857 frags
[i
- 1].size
+= copy
;
1859 if (i
>= frag_max
) {
1860 pr_warn("too many pages %u, dlen %u.\n",
1865 frags
[i
].page
= page
;
1866 frags
[i
].offset
= sg
->offset
+ sgoffset
;
1867 frags
[i
].size
= copy
;
1878 int cxgbi_conn_alloc_pdu(struct iscsi_task
*task
, u8 opcode
)
1880 struct iscsi_tcp_conn
*tcp_conn
= task
->conn
->dd_data
;
1881 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
1882 struct cxgbi_device
*cdev
= cconn
->chba
->cdev
;
1883 struct iscsi_conn
*conn
= task
->conn
;
1884 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
1885 struct cxgbi_task_data
*tdata
= iscsi_task_cxgbi_data(task
);
1886 struct scsi_cmnd
*sc
= task
->sc
;
1887 struct cxgbi_sock
*csk
= cconn
->cep
->csk
;
1888 struct net_device
*ndev
= cdev
->ports
[csk
->port_id
];
1889 int headroom
= SKB_TX_ISCSI_PDU_HEADER_MAX
;
1891 tcp_task
->dd_data
= tdata
;
1894 if (SKB_MAX_HEAD(cdev
->skb_tx_rsvd
) > (512 * MAX_SKB_FRAGS
) &&
1895 (opcode
== ISCSI_OP_SCSI_DATA_OUT
||
1896 (opcode
== ISCSI_OP_SCSI_CMD
&&
1897 sc
->sc_data_direction
== DMA_TO_DEVICE
)))
1898 /* data could goes into skb head */
1899 headroom
+= min_t(unsigned int,
1900 SKB_MAX_HEAD(cdev
->skb_tx_rsvd
),
1901 conn
->max_xmit_dlength
);
1903 tdata
->skb
= alloc_skb(cdev
->skb_tx_rsvd
+ headroom
, GFP_ATOMIC
);
1905 ndev
->stats
.tx_dropped
++;
1909 skb_reserve(tdata
->skb
, cdev
->skb_tx_rsvd
);
1912 task
->hdr
= (struct iscsi_hdr
*)tdata
->skb
->data
;
1914 task
->hdr
= kzalloc(SKB_TX_ISCSI_PDU_HEADER_MAX
, GFP_ATOMIC
);
1916 __kfree_skb(tdata
->skb
);
1918 ndev
->stats
.tx_dropped
++;
1922 task
->hdr_max
= SKB_TX_ISCSI_PDU_HEADER_MAX
; /* BHS + AHS */
1924 /* data_out uses scsi_cmd's itt */
1925 if (opcode
!= ISCSI_OP_SCSI_DATA_OUT
)
1926 task_reserve_itt(task
, &task
->hdr
->itt
);
1928 log_debug(1 << CXGBI_DBG_ISCSI
| 1 << CXGBI_DBG_PDU_TX
,
1929 "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n",
1930 task
, opcode
, tdata
->skb
, cdev
->skb_tx_rsvd
, headroom
,
1931 conn
->max_xmit_dlength
, ntohl(task
->hdr
->itt
));
1935 EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu
);
1937 static inline void tx_skb_setmode(struct sk_buff
*skb
, int hcrc
, int dcrc
)
1946 cxgbi_skcb_ulp_mode(skb
) = (ULP2_MODE_ISCSI
<< 4) | submode
;
1948 cxgbi_skcb_ulp_mode(skb
) = 0;
1951 int cxgbi_conn_init_pdu(struct iscsi_task
*task
, unsigned int offset
,
1954 struct iscsi_conn
*conn
= task
->conn
;
1955 struct cxgbi_task_data
*tdata
= iscsi_task_cxgbi_data(task
);
1956 struct sk_buff
*skb
= tdata
->skb
;
1957 unsigned int datalen
= count
;
1958 int i
, padlen
= iscsi_padding(count
);
1961 log_debug(1 << CXGBI_DBG_ISCSI
| 1 << CXGBI_DBG_PDU_TX
,
1962 "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n",
1963 task
, task
->sc
, skb
, (*skb
->data
) & ISCSI_OPCODE_MASK
,
1964 ntohl(task
->cmdsn
), ntohl(task
->hdr
->itt
), offset
, count
);
1966 skb_put(skb
, task
->hdr_len
);
1967 tx_skb_setmode(skb
, conn
->hdrdgst_en
, datalen
? conn
->datadgst_en
: 0);
1972 struct scsi_data_buffer
*sdb
= &task
->sc
->sdb
;
1973 struct scatterlist
*sg
= NULL
;
1976 tdata
->offset
= offset
;
1977 tdata
->count
= count
;
1978 err
= sgl_seek_offset(
1979 sdb
->table
.sgl
, sdb
->table
.nents
,
1980 tdata
->offset
, &tdata
->sgoffset
, &sg
);
1982 pr_warn("tpdu, sgl %u, bad offset %u/%u.\n",
1983 sdb
->table
.nents
, tdata
->offset
, sdb
->length
);
1986 err
= sgl_read_to_frags(sg
, tdata
->sgoffset
, tdata
->count
,
1987 tdata
->frags
, MAX_PDU_FRAGS
);
1989 pr_warn("tpdu, sgl %u, bad offset %u + %u.\n",
1990 sdb
->table
.nents
, tdata
->offset
, tdata
->count
);
1993 tdata
->nr_frags
= err
;
1995 if (tdata
->nr_frags
> MAX_SKB_FRAGS
||
1996 (padlen
&& tdata
->nr_frags
== MAX_SKB_FRAGS
)) {
1997 char *dst
= skb
->data
+ task
->hdr_len
;
1998 struct page_frag
*frag
= tdata
->frags
;
2000 /* data fits in the skb's headroom */
2001 for (i
= 0; i
< tdata
->nr_frags
; i
++, frag
++) {
2002 char *src
= kmap_atomic(frag
->page
);
2004 memcpy(dst
, src
+frag
->offset
, frag
->size
);
2009 memset(dst
, 0, padlen
);
2012 skb_put(skb
, count
+ padlen
);
2014 /* data fit into frag_list */
2015 for (i
= 0; i
< tdata
->nr_frags
; i
++) {
2016 __skb_fill_page_desc(skb
, i
,
2017 tdata
->frags
[i
].page
,
2018 tdata
->frags
[i
].offset
,
2019 tdata
->frags
[i
].size
);
2020 skb_frag_ref(skb
, i
);
2022 skb_shinfo(skb
)->nr_frags
= tdata
->nr_frags
;
2024 skb
->data_len
+= count
;
2025 skb
->truesize
+= count
;
2029 pg
= virt_to_page(task
->data
);
2032 skb_fill_page_desc(skb
, 0, pg
, offset_in_page(task
->data
),
2035 skb
->data_len
+= count
;
2036 skb
->truesize
+= count
;
2040 i
= skb_shinfo(skb
)->nr_frags
;
2041 skb_fill_page_desc(skb
, skb_shinfo(skb
)->nr_frags
,
2042 virt_to_page(padding
), offset_in_page(padding
),
2045 skb
->data_len
+= padlen
;
2046 skb
->truesize
+= padlen
;
2052 EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu
);
2054 int cxgbi_conn_xmit_pdu(struct iscsi_task
*task
)
2056 struct iscsi_tcp_conn
*tcp_conn
= task
->conn
->dd_data
;
2057 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
2058 struct cxgbi_task_data
*tdata
= iscsi_task_cxgbi_data(task
);
2059 struct cxgbi_task_tag_info
*ttinfo
= &tdata
->ttinfo
;
2060 struct sk_buff
*skb
= tdata
->skb
;
2061 struct cxgbi_sock
*csk
= NULL
;
2062 unsigned int datalen
;
2066 log_debug(1 << CXGBI_DBG_ISCSI
| 1 << CXGBI_DBG_PDU_TX
,
2067 "task 0x%p\n", task
);
2071 if (cconn
&& cconn
->cep
)
2072 csk
= cconn
->cep
->csk
;
2074 log_debug(1 << CXGBI_DBG_ISCSI
| 1 << CXGBI_DBG_PDU_TX
,
2075 "task 0x%p, csk gone.\n", task
);
2080 datalen
= skb
->data_len
;
2082 /* write ppod first if using ofldq to write ppod */
2083 if (ttinfo
->flags
& CXGBI_PPOD_INFO_FLAG_VALID
) {
2084 struct cxgbi_ppm
*ppm
= csk
->cdev
->cdev2ppm(csk
->cdev
);
2086 ttinfo
->flags
&= ~CXGBI_PPOD_INFO_FLAG_VALID
;
2087 if (csk
->cdev
->csk_ddp_set_map(ppm
, csk
, ttinfo
) < 0)
2088 pr_err("task 0x%p, ppod writing using ofldq failed.\n",
2090 /* continue. Let fl get the data */
2094 memcpy(skb
->data
, task
->hdr
, SKB_TX_ISCSI_PDU_HEADER_MAX
);
2096 err
= cxgbi_sock_send_pdus(cconn
->cep
->csk
, skb
);
2100 log_debug(1 << CXGBI_DBG_PDU_TX
,
2101 "task 0x%p,0x%p, skb 0x%p, len %u/%u, rv %d.\n",
2102 task
, task
->sc
, skb
, skb
->len
, skb
->data_len
, err
);
2104 if (task
->conn
->hdrdgst_en
)
2105 pdulen
+= ISCSI_DIGEST_SIZE
;
2107 if (datalen
&& task
->conn
->datadgst_en
)
2108 pdulen
+= ISCSI_DIGEST_SIZE
;
2110 task
->conn
->txdata_octets
+= pdulen
;
2114 if (err
== -EAGAIN
|| err
== -ENOBUFS
) {
2115 log_debug(1 << CXGBI_DBG_PDU_TX
,
2116 "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n",
2117 task
, skb
, skb
->len
, skb
->data_len
, err
);
2118 /* reset skb to send when we are called again */
2123 log_debug(1 << CXGBI_DBG_ISCSI
| 1 << CXGBI_DBG_PDU_TX
,
2124 "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
2125 task
->itt
, skb
, skb
->len
, skb
->data_len
, err
);
2129 iscsi_conn_printk(KERN_ERR
, task
->conn
, "xmit err %d.\n", err
);
2130 iscsi_conn_failure(task
->conn
, ISCSI_ERR_XMIT_FAILED
);
2133 EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu
);
2135 void cxgbi_cleanup_task(struct iscsi_task
*task
)
2137 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
2138 struct cxgbi_task_data
*tdata
= iscsi_task_cxgbi_data(task
);
2140 if (!tcp_task
|| !tdata
|| (tcp_task
->dd_data
!= tdata
)) {
2141 pr_info("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
2142 task
, task
->sc
, tcp_task
,
2143 tcp_task
? tcp_task
->dd_data
: NULL
, tdata
);
2147 log_debug(1 << CXGBI_DBG_ISCSI
,
2148 "task 0x%p, skb 0x%p, itt 0x%x.\n",
2149 task
, tdata
->skb
, task
->hdr_itt
);
2151 tcp_task
->dd_data
= NULL
;
2157 /* never reached the xmit task callout */
2159 __kfree_skb(tdata
->skb
);
2163 task_release_itt(task
, task
->hdr_itt
);
2164 memset(tdata
, 0, sizeof(*tdata
));
2166 iscsi_tcp_cleanup_task(task
);
2168 EXPORT_SYMBOL_GPL(cxgbi_cleanup_task
);
2170 void cxgbi_get_conn_stats(struct iscsi_cls_conn
*cls_conn
,
2171 struct iscsi_stats
*stats
)
2173 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
2175 stats
->txdata_octets
= conn
->txdata_octets
;
2176 stats
->rxdata_octets
= conn
->rxdata_octets
;
2177 stats
->scsicmd_pdus
= conn
->scsicmd_pdus_cnt
;
2178 stats
->dataout_pdus
= conn
->dataout_pdus_cnt
;
2179 stats
->scsirsp_pdus
= conn
->scsirsp_pdus_cnt
;
2180 stats
->datain_pdus
= conn
->datain_pdus_cnt
;
2181 stats
->r2t_pdus
= conn
->r2t_pdus_cnt
;
2182 stats
->tmfcmd_pdus
= conn
->tmfcmd_pdus_cnt
;
2183 stats
->tmfrsp_pdus
= conn
->tmfrsp_pdus_cnt
;
2184 stats
->digest_err
= 0;
2185 stats
->timeout_err
= 0;
2186 stats
->custom_length
= 1;
2187 strcpy(stats
->custom
[0].desc
, "eh_abort_cnt");
2188 stats
->custom
[0].value
= conn
->eh_abort_cnt
;
2190 EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats
);
2192 static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn
*conn
)
2194 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
2195 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
2196 struct cxgbi_device
*cdev
= cconn
->chba
->cdev
;
2197 unsigned int headroom
= SKB_MAX_HEAD(cdev
->skb_tx_rsvd
);
2198 unsigned int max_def
= 512 * MAX_SKB_FRAGS
;
2199 unsigned int max
= max(max_def
, headroom
);
2201 max
= min(cconn
->chba
->cdev
->tx_max_size
, max
);
2202 if (conn
->max_xmit_dlength
)
2203 conn
->max_xmit_dlength
= min(conn
->max_xmit_dlength
, max
);
2205 conn
->max_xmit_dlength
= max
;
2206 cxgbi_align_pdu_size(conn
->max_xmit_dlength
);
2211 static int cxgbi_conn_max_recv_dlength(struct iscsi_conn
*conn
)
2213 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
2214 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
2215 unsigned int max
= cconn
->chba
->cdev
->rx_max_size
;
2217 cxgbi_align_pdu_size(max
);
2219 if (conn
->max_recv_dlength
) {
2220 if (conn
->max_recv_dlength
> max
) {
2221 pr_err("MaxRecvDataSegmentLength %u > %u.\n",
2222 conn
->max_recv_dlength
, max
);
2225 conn
->max_recv_dlength
= min(conn
->max_recv_dlength
, max
);
2226 cxgbi_align_pdu_size(conn
->max_recv_dlength
);
2228 conn
->max_recv_dlength
= max
;
2233 int cxgbi_set_conn_param(struct iscsi_cls_conn
*cls_conn
,
2234 enum iscsi_param param
, char *buf
, int buflen
)
2236 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
2237 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
2238 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
2239 struct cxgbi_sock
*csk
= cconn
->cep
->csk
;
2242 log_debug(1 << CXGBI_DBG_ISCSI
,
2243 "cls_conn 0x%p, param %d, buf(%d) %s.\n",
2244 cls_conn
, param
, buflen
, buf
);
2247 case ISCSI_PARAM_HDRDGST_EN
:
2248 err
= iscsi_set_param(cls_conn
, param
, buf
, buflen
);
2249 if (!err
&& conn
->hdrdgst_en
)
2250 err
= csk
->cdev
->csk_ddp_setup_digest(csk
, csk
->tid
,
2254 case ISCSI_PARAM_DATADGST_EN
:
2255 err
= iscsi_set_param(cls_conn
, param
, buf
, buflen
);
2256 if (!err
&& conn
->datadgst_en
)
2257 err
= csk
->cdev
->csk_ddp_setup_digest(csk
, csk
->tid
,
2261 case ISCSI_PARAM_MAX_R2T
:
2262 return iscsi_tcp_set_max_r2t(conn
, buf
);
2263 case ISCSI_PARAM_MAX_RECV_DLENGTH
:
2264 err
= iscsi_set_param(cls_conn
, param
, buf
, buflen
);
2266 err
= cxgbi_conn_max_recv_dlength(conn
);
2268 case ISCSI_PARAM_MAX_XMIT_DLENGTH
:
2269 err
= iscsi_set_param(cls_conn
, param
, buf
, buflen
);
2271 err
= cxgbi_conn_max_xmit_dlength(conn
);
2274 return iscsi_set_param(cls_conn
, param
, buf
, buflen
);
2278 EXPORT_SYMBOL_GPL(cxgbi_set_conn_param
);
2280 static inline int csk_print_port(struct cxgbi_sock
*csk
, char *buf
)
2284 cxgbi_sock_get(csk
);
2285 len
= sprintf(buf
, "%hu\n", ntohs(csk
->daddr
.sin_port
));
2286 cxgbi_sock_put(csk
);
2291 static inline int csk_print_ip(struct cxgbi_sock
*csk
, char *buf
)
2295 cxgbi_sock_get(csk
);
2296 if (csk
->csk_family
== AF_INET
)
2297 len
= sprintf(buf
, "%pI4",
2298 &csk
->daddr
.sin_addr
.s_addr
);
2300 len
= sprintf(buf
, "%pI6",
2301 &csk
->daddr6
.sin6_addr
);
2303 cxgbi_sock_put(csk
);
2308 int cxgbi_get_ep_param(struct iscsi_endpoint
*ep
, enum iscsi_param param
,
2311 struct cxgbi_endpoint
*cep
= ep
->dd_data
;
2312 struct cxgbi_sock
*csk
;
2314 log_debug(1 << CXGBI_DBG_ISCSI
,
2315 "cls_conn 0x%p, param %d.\n", ep
, param
);
2318 case ISCSI_PARAM_CONN_PORT
:
2319 case ISCSI_PARAM_CONN_ADDRESS
:
2327 return iscsi_conn_get_addr_param((struct sockaddr_storage
*)
2328 &csk
->daddr
, param
, buf
);
2334 EXPORT_SYMBOL_GPL(cxgbi_get_ep_param
);
2336 struct iscsi_cls_conn
*
2337 cxgbi_create_conn(struct iscsi_cls_session
*cls_session
, u32 cid
)
2339 struct iscsi_cls_conn
*cls_conn
;
2340 struct iscsi_conn
*conn
;
2341 struct iscsi_tcp_conn
*tcp_conn
;
2342 struct cxgbi_conn
*cconn
;
2344 cls_conn
= iscsi_tcp_conn_setup(cls_session
, sizeof(*cconn
), cid
);
2348 conn
= cls_conn
->dd_data
;
2349 tcp_conn
= conn
->dd_data
;
2350 cconn
= tcp_conn
->dd_data
;
2351 cconn
->iconn
= conn
;
2353 log_debug(1 << CXGBI_DBG_ISCSI
,
2354 "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n",
2355 cid
, cid
, cls_session
, cls_conn
, conn
, tcp_conn
, cconn
);
2359 EXPORT_SYMBOL_GPL(cxgbi_create_conn
);
2361 int cxgbi_bind_conn(struct iscsi_cls_session
*cls_session
,
2362 struct iscsi_cls_conn
*cls_conn
,
2363 u64 transport_eph
, int is_leading
)
2365 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
2366 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
2367 struct cxgbi_conn
*cconn
= tcp_conn
->dd_data
;
2368 struct cxgbi_ppm
*ppm
;
2369 struct iscsi_endpoint
*ep
;
2370 struct cxgbi_endpoint
*cep
;
2371 struct cxgbi_sock
*csk
;
2374 ep
= iscsi_lookup_endpoint(transport_eph
);
2378 /* setup ddp pagesize */
2382 ppm
= csk
->cdev
->cdev2ppm(csk
->cdev
);
2383 err
= csk
->cdev
->csk_ddp_setup_pgidx(csk
, csk
->tid
,
2384 ppm
->tformat
.pgsz_idx_dflt
);
2388 err
= iscsi_conn_bind(cls_session
, cls_conn
, is_leading
);
2392 /* calculate the tag idx bits needed for this conn based on cmds_max */
2393 cconn
->task_idx_bits
= (__ilog2_u32(conn
->session
->cmds_max
- 1)) + 1;
2395 write_lock_bh(&csk
->callback_lock
);
2396 csk
->user_data
= conn
;
2397 cconn
->chba
= cep
->chba
;
2400 write_unlock_bh(&csk
->callback_lock
);
2402 cxgbi_conn_max_xmit_dlength(conn
);
2403 cxgbi_conn_max_recv_dlength(conn
);
2405 log_debug(1 << CXGBI_DBG_ISCSI
,
2406 "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n",
2407 cls_session
, cls_conn
, ep
, cconn
, csk
);
2408 /* init recv engine */
2409 iscsi_tcp_hdr_recv_prep(tcp_conn
);
2413 EXPORT_SYMBOL_GPL(cxgbi_bind_conn
);
2415 struct iscsi_cls_session
*cxgbi_create_session(struct iscsi_endpoint
*ep
,
2416 u16 cmds_max
, u16 qdepth
,
2419 struct cxgbi_endpoint
*cep
;
2420 struct cxgbi_hba
*chba
;
2421 struct Scsi_Host
*shost
;
2422 struct iscsi_cls_session
*cls_session
;
2423 struct iscsi_session
*session
;
2426 pr_err("missing endpoint.\n");
2432 shost
= chba
->shost
;
2434 BUG_ON(chba
!= iscsi_host_priv(shost
));
2436 cls_session
= iscsi_session_setup(chba
->cdev
->itp
, shost
,
2438 sizeof(struct iscsi_tcp_task
) +
2439 sizeof(struct cxgbi_task_data
),
2440 initial_cmdsn
, ISCSI_MAX_TARGET
);
2444 session
= cls_session
->dd_data
;
2445 if (iscsi_tcp_r2tpool_alloc(session
))
2446 goto remove_session
;
2448 log_debug(1 << CXGBI_DBG_ISCSI
,
2449 "ep 0x%p, cls sess 0x%p.\n", ep
, cls_session
);
2453 iscsi_session_teardown(cls_session
);
2456 EXPORT_SYMBOL_GPL(cxgbi_create_session
);
2458 void cxgbi_destroy_session(struct iscsi_cls_session
*cls_session
)
2460 log_debug(1 << CXGBI_DBG_ISCSI
,
2461 "cls sess 0x%p.\n", cls_session
);
2463 iscsi_tcp_r2tpool_free(cls_session
->dd_data
);
2464 iscsi_session_teardown(cls_session
);
2466 EXPORT_SYMBOL_GPL(cxgbi_destroy_session
);
2468 int cxgbi_set_host_param(struct Scsi_Host
*shost
, enum iscsi_host_param param
,
2469 char *buf
, int buflen
)
2471 struct cxgbi_hba
*chba
= iscsi_host_priv(shost
);
2474 shost_printk(KERN_ERR
, shost
, "Could not get host param. "
2475 "netdev for host not set.\n");
2479 log_debug(1 << CXGBI_DBG_ISCSI
,
2480 "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n",
2481 shost
, chba
, chba
->ndev
->name
, param
, buflen
, buf
);
2484 case ISCSI_HOST_PARAM_IPADDRESS
:
2486 __be32 addr
= in_aton(buf
);
2487 log_debug(1 << CXGBI_DBG_ISCSI
,
2488 "hba %s, req. ipv4 %pI4.\n", chba
->ndev
->name
, &addr
);
2489 cxgbi_set_iscsi_ipv4(chba
, addr
);
2492 case ISCSI_HOST_PARAM_HWADDRESS
:
2493 case ISCSI_HOST_PARAM_NETDEV_NAME
:
2496 return iscsi_host_set_param(shost
, param
, buf
, buflen
);
2499 EXPORT_SYMBOL_GPL(cxgbi_set_host_param
);
2501 int cxgbi_get_host_param(struct Scsi_Host
*shost
, enum iscsi_host_param param
,
2504 struct cxgbi_hba
*chba
= iscsi_host_priv(shost
);
2508 shost_printk(KERN_ERR
, shost
, "Could not get host param. "
2509 "netdev for host not set.\n");
2513 log_debug(1 << CXGBI_DBG_ISCSI
,
2514 "shost 0x%p, hba 0x%p,%s, param %d.\n",
2515 shost
, chba
, chba
->ndev
->name
, param
);
2518 case ISCSI_HOST_PARAM_HWADDRESS
:
2519 len
= sysfs_format_mac(buf
, chba
->ndev
->dev_addr
, 6);
2521 case ISCSI_HOST_PARAM_NETDEV_NAME
:
2522 len
= sprintf(buf
, "%s\n", chba
->ndev
->name
);
2524 case ISCSI_HOST_PARAM_IPADDRESS
:
2526 struct cxgbi_sock
*csk
= find_sock_on_port(chba
->cdev
,
2529 len
= sprintf(buf
, "%pIS",
2530 (struct sockaddr
*)&csk
->saddr
);
2532 log_debug(1 << CXGBI_DBG_ISCSI
,
2533 "hba %s, addr %s.\n", chba
->ndev
->name
, buf
);
2537 return iscsi_host_get_param(shost
, param
, buf
);
2542 EXPORT_SYMBOL_GPL(cxgbi_get_host_param
);
2544 struct iscsi_endpoint
*cxgbi_ep_connect(struct Scsi_Host
*shost
,
2545 struct sockaddr
*dst_addr
,
2548 struct iscsi_endpoint
*ep
;
2549 struct cxgbi_endpoint
*cep
;
2550 struct cxgbi_hba
*hba
= NULL
;
2551 struct cxgbi_sock
*csk
;
2555 log_debug(1 << CXGBI_DBG_ISCSI
| 1 << CXGBI_DBG_SOCK
,
2556 "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n",
2557 shost
, non_blocking
, dst_addr
);
2560 hba
= iscsi_host_priv(shost
);
2562 pr_info("shost 0x%p, priv NULL.\n", shost
);
2568 if (dst_addr
->sa_family
== AF_INET
) {
2569 csk
= cxgbi_check_route(dst_addr
, ifindex
);
2570 #if IS_ENABLED(CONFIG_IPV6)
2571 } else if (dst_addr
->sa_family
== AF_INET6
) {
2572 csk
= cxgbi_check_route6(dst_addr
, ifindex
);
2575 pr_info("address family 0x%x NOT supported.\n",
2576 dst_addr
->sa_family
);
2577 err
= -EAFNOSUPPORT
;
2578 return (struct iscsi_endpoint
*)ERR_PTR(err
);
2582 return (struct iscsi_endpoint
*)csk
;
2583 cxgbi_sock_get(csk
);
2586 hba
= csk
->cdev
->hbas
[csk
->port_id
];
2587 else if (hba
!= csk
->cdev
->hbas
[csk
->port_id
]) {
2588 if (ifindex
!= hba
->ndev
->ifindex
) {
2589 cxgbi_sock_put(csk
);
2590 cxgbi_sock_closed(csk
);
2591 ifindex
= hba
->ndev
->ifindex
;
2595 pr_info("Could not connect through requested host %u"
2596 "hba 0x%p != 0x%p (%u).\n",
2597 shost
->host_no
, hba
,
2598 csk
->cdev
->hbas
[csk
->port_id
], csk
->port_id
);
2603 err
= sock_get_port(csk
);
2607 cxgbi_sock_set_state(csk
, CTP_CONNECTING
);
2608 err
= csk
->cdev
->csk_init_act_open(csk
);
2612 if (cxgbi_sock_is_closing(csk
)) {
2614 pr_info("csk 0x%p is closing.\n", csk
);
2618 ep
= iscsi_create_endpoint(sizeof(*cep
));
2621 pr_info("iscsi alloc ep, OOM.\n");
2629 log_debug(1 << CXGBI_DBG_ISCSI
| 1 << CXGBI_DBG_SOCK
,
2630 "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n",
2631 ep
, cep
, csk
, hba
, hba
->ndev
->name
);
2635 cxgbi_sock_put(csk
);
2636 cxgbi_sock_closed(csk
);
2638 return ERR_PTR(err
);
2640 EXPORT_SYMBOL_GPL(cxgbi_ep_connect
);
2642 int cxgbi_ep_poll(struct iscsi_endpoint
*ep
, int timeout_ms
)
2644 struct cxgbi_endpoint
*cep
= ep
->dd_data
;
2645 struct cxgbi_sock
*csk
= cep
->csk
;
2647 if (!cxgbi_sock_is_established(csk
))
2651 EXPORT_SYMBOL_GPL(cxgbi_ep_poll
);
2653 void cxgbi_ep_disconnect(struct iscsi_endpoint
*ep
)
2655 struct cxgbi_endpoint
*cep
= ep
->dd_data
;
2656 struct cxgbi_conn
*cconn
= cep
->cconn
;
2657 struct cxgbi_sock
*csk
= cep
->csk
;
2659 log_debug(1 << CXGBI_DBG_ISCSI
| 1 << CXGBI_DBG_SOCK
,
2660 "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n",
2661 ep
, cep
, cconn
, csk
, csk
->state
, csk
->flags
);
2663 if (cconn
&& cconn
->iconn
) {
2664 iscsi_suspend_tx(cconn
->iconn
);
2665 write_lock_bh(&csk
->callback_lock
);
2666 cep
->csk
->user_data
= NULL
;
2668 write_unlock_bh(&csk
->callback_lock
);
2670 iscsi_destroy_endpoint(ep
);
2672 if (likely(csk
->state
>= CTP_ESTABLISHED
))
2673 need_active_close(csk
);
2675 cxgbi_sock_closed(csk
);
2677 cxgbi_sock_put(csk
);
2679 EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect
);
2681 int cxgbi_iscsi_init(struct iscsi_transport
*itp
,
2682 struct scsi_transport_template
**stt
)
2684 *stt
= iscsi_register_transport(itp
);
2686 pr_err("unable to register %s transport 0x%p.\n",
2690 log_debug(1 << CXGBI_DBG_ISCSI
,
2691 "%s, registered iscsi transport 0x%p.\n",
2695 EXPORT_SYMBOL_GPL(cxgbi_iscsi_init
);
2697 void cxgbi_iscsi_cleanup(struct iscsi_transport
*itp
,
2698 struct scsi_transport_template
**stt
)
2701 log_debug(1 << CXGBI_DBG_ISCSI
,
2702 "de-register transport 0x%p, %s, stt 0x%p.\n",
2703 itp
, itp
->name
, *stt
);
2705 iscsi_unregister_transport(itp
);
2708 EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup
);
2710 umode_t
cxgbi_attr_is_visible(int param_type
, int param
)
2712 switch (param_type
) {
2713 case ISCSI_HOST_PARAM
:
2715 case ISCSI_HOST_PARAM_NETDEV_NAME
:
2716 case ISCSI_HOST_PARAM_HWADDRESS
:
2717 case ISCSI_HOST_PARAM_IPADDRESS
:
2718 case ISCSI_HOST_PARAM_INITIATOR_NAME
:
2725 case ISCSI_PARAM_MAX_RECV_DLENGTH
:
2726 case ISCSI_PARAM_MAX_XMIT_DLENGTH
:
2727 case ISCSI_PARAM_HDRDGST_EN
:
2728 case ISCSI_PARAM_DATADGST_EN
:
2729 case ISCSI_PARAM_CONN_ADDRESS
:
2730 case ISCSI_PARAM_CONN_PORT
:
2731 case ISCSI_PARAM_EXP_STATSN
:
2732 case ISCSI_PARAM_PERSISTENT_ADDRESS
:
2733 case ISCSI_PARAM_PERSISTENT_PORT
:
2734 case ISCSI_PARAM_PING_TMO
:
2735 case ISCSI_PARAM_RECV_TMO
:
2736 case ISCSI_PARAM_INITIAL_R2T_EN
:
2737 case ISCSI_PARAM_MAX_R2T
:
2738 case ISCSI_PARAM_IMM_DATA_EN
:
2739 case ISCSI_PARAM_FIRST_BURST
:
2740 case ISCSI_PARAM_MAX_BURST
:
2741 case ISCSI_PARAM_PDU_INORDER_EN
:
2742 case ISCSI_PARAM_DATASEQ_INORDER_EN
:
2743 case ISCSI_PARAM_ERL
:
2744 case ISCSI_PARAM_TARGET_NAME
:
2745 case ISCSI_PARAM_TPGT
:
2746 case ISCSI_PARAM_USERNAME
:
2747 case ISCSI_PARAM_PASSWORD
:
2748 case ISCSI_PARAM_USERNAME_IN
:
2749 case ISCSI_PARAM_PASSWORD_IN
:
2750 case ISCSI_PARAM_FAST_ABORT
:
2751 case ISCSI_PARAM_ABORT_TMO
:
2752 case ISCSI_PARAM_LU_RESET_TMO
:
2753 case ISCSI_PARAM_TGT_RESET_TMO
:
2754 case ISCSI_PARAM_IFACE_NAME
:
2755 case ISCSI_PARAM_INITIATOR_NAME
:
2764 EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible
);
2766 static int __init
libcxgbi_init_module(void)
2768 pr_info("%s", version
);
2770 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff
, cb
) <
2771 sizeof(struct cxgbi_skb_cb
));
2775 static void __exit
libcxgbi_exit_module(void)
2777 cxgbi_device_unregister_all(0xFF);
2781 module_init(libcxgbi_init_module
);
2782 module_exit(libcxgbi_exit_module
);