]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/scsi/cxgbi/libcxgbi.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cxgb4 : Fix build failure in cxgb4 when ipv6 is disabled/not in-built
[mirror_ubuntu-artful-kernel.git] / drivers / scsi / cxgbi / libcxgbi.c
CommitLineData
9ba682f0 1/*
2 * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver.
3 *
4 * Copyright (c) 2010 Chelsio Communications, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 *
10 * Written by: Karen Xie (kxie@chelsio.com)
11 * Written by: Rakesh Ranjan (rranjan@chelsio.com)
12 */
13
14#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
15
16#include <linux/skbuff.h>
17#include <linux/crypto.h>
18#include <linux/scatterlist.h>
19#include <linux/pci.h>
20#include <scsi/scsi.h>
21#include <scsi/scsi_cmnd.h>
22#include <scsi/scsi_host.h>
23#include <linux/if_vlan.h>
24#include <linux/inet.h>
25#include <net/dst.h>
26#include <net/route.h>
fc8d0590
AB		 27#include <net/ipv6.h>
28#include <net/ip6_route.h>
29#include <net/addrconf.h>
30
9ba682f0 31#include <linux/inetdevice.h> /* ip_dev_find */
acf3368f 32#include <linux/module.h>
9ba682f0 33#include <net/tcp.h>
34
35static unsigned int dbg_level;
36
37#include "libcxgbi.h"
38
39#define DRV_MODULE_NAME "libcxgbi"
40#define DRV_MODULE_DESC "Chelsio iSCSI driver library"
41#define DRV_MODULE_VERSION "0.9.0"
42#define DRV_MODULE_RELDATE "Jun. 2010"
43
44MODULE_AUTHOR("Chelsio Communications, Inc.");
45MODULE_DESCRIPTION(DRV_MODULE_DESC);
46MODULE_VERSION(DRV_MODULE_VERSION);
47MODULE_LICENSE("GPL");
48
49module_param(dbg_level, uint, 0644);
50MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)");
51
52
53/*
54 * cxgbi device management
55 * maintains a list of the cxgbi devices
56 */
57static LIST_HEAD(cdev_list);
58static DEFINE_MUTEX(cdev_mutex);
59
576b5863
AB		 60static LIST_HEAD(cdev_rcu_list);
61static DEFINE_SPINLOCK(cdev_rcu_lock);
62
9ba682f0 63int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base,
64 unsigned int max_conn)
65{
66 struct cxgbi_ports_map *pmap = &cdev->pmap;
67
68 pmap->port_csk = cxgbi_alloc_big_mem(max_conn *
69 sizeof(struct cxgbi_sock *),
70 GFP_KERNEL);
71 if (!pmap->port_csk) {
72 pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn);
73 return -ENOMEM;
74 }
75
76 pmap->max_connect = max_conn;
77 pmap->sport_base = base;
78 spin_lock_init(&pmap->lock);
79 return 0;
80}
81EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create);
82
83void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev)
84{
85 struct cxgbi_ports_map *pmap = &cdev->pmap;
86 struct cxgbi_sock *csk;
87 int i;
88
89 for (i = 0; i < pmap->max_connect; i++) {
90 if (pmap->port_csk[i]) {
91 csk = pmap->port_csk[i];
92 pmap->port_csk[i] = NULL;
93 log_debug(1 << CXGBI_DBG_SOCK,
94 "csk 0x%p, cdev 0x%p, offload down.\n",
95 csk, cdev);
96 spin_lock_bh(&csk->lock);
97 cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN);
98 cxgbi_sock_closed(csk);
99 spin_unlock_bh(&csk->lock);
100 cxgbi_sock_put(csk);
101 }
102 }
103}
104EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup);
105
106static inline void cxgbi_device_destroy(struct cxgbi_device *cdev)
107{
108 log_debug(1 << CXGBI_DBG_DEV,
109 "cdev 0x%p, p# %u.\n", cdev, cdev->nports);
110 cxgbi_hbas_remove(cdev);
111 cxgbi_device_portmap_cleanup(cdev);
112 if (cdev->dev_ddp_cleanup)
113 cdev->dev_ddp_cleanup(cdev);
114 else
115 cxgbi_ddp_cleanup(cdev);
116 if (cdev->ddp)
117 cxgbi_ddp_cleanup(cdev);
118 if (cdev->pmap.max_connect)
119 cxgbi_free_big_mem(cdev->pmap.port_csk);
120 kfree(cdev);
121}
122
123struct cxgbi_device *cxgbi_device_register(unsigned int extra,
124 unsigned int nports)
125{
126 struct cxgbi_device *cdev;
127
128 cdev = kzalloc(sizeof(*cdev) + extra + nports *
129 (sizeof(struct cxgbi_hba *) +
130 sizeof(struct net_device *)),
131 GFP_KERNEL);
132 if (!cdev) {
133 pr_warn("nport %d, OOM.\n", nports);
134 return NULL;
135 }
136 cdev->ports = (struct net_device **)(cdev + 1);
137 cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports *
138 sizeof(struct net_device *));
139 if (extra)
140 cdev->dd_data = ((char *)cdev->hbas) +
141 nports * sizeof(struct cxgbi_hba *);
142 spin_lock_init(&cdev->pmap.lock);
143
144 mutex_lock(&cdev_mutex);
145 list_add_tail(&cdev->list_head, &cdev_list);
146 mutex_unlock(&cdev_mutex);
147
576b5863
AB		 148 spin_lock(&cdev_rcu_lock);
149 list_add_tail_rcu(&cdev->rcu_node, &cdev_rcu_list);
150 spin_unlock(&cdev_rcu_lock);
151
9ba682f0 152 log_debug(1 << CXGBI_DBG_DEV,
153 "cdev 0x%p, p# %u.\n", cdev, nports);
154 return cdev;
155}
156EXPORT_SYMBOL_GPL(cxgbi_device_register);
157
158void cxgbi_device_unregister(struct cxgbi_device *cdev)
159{
160 log_debug(1 << CXGBI_DBG_DEV,
161 "cdev 0x%p, p# %u,%s.\n",
162 cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : "");
576b5863 163
9ba682f0 164 mutex_lock(&cdev_mutex);
165 list_del(&cdev->list_head);
166 mutex_unlock(&cdev_mutex);
576b5863
AB		 167
168 spin_lock(&cdev_rcu_lock);
169 list_del_rcu(&cdev->rcu_node);
170 spin_unlock(&cdev_rcu_lock);
171 synchronize_rcu();
172
9ba682f0 173 cxgbi_device_destroy(cdev);
174}
175EXPORT_SYMBOL_GPL(cxgbi_device_unregister);
176
177void cxgbi_device_unregister_all(unsigned int flag)
178{
179 struct cxgbi_device *cdev, *tmp;
180
181 mutex_lock(&cdev_mutex);
182 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
183 if ((cdev->flags & flag) == flag) {
576b5863
AB		 184 mutex_unlock(&cdev_mutex);
185 cxgbi_device_unregister(cdev);
186 mutex_lock(&cdev_mutex);
9ba682f0 187 }
188 }
189 mutex_unlock(&cdev_mutex);
190}
191EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all);
192
193struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev)
194{
195 struct cxgbi_device *cdev, *tmp;
196
197 mutex_lock(&cdev_mutex);
198 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
199 if (cdev->lldev == lldev) {
200 mutex_unlock(&cdev_mutex);
201 return cdev;
202 }
203 }
204 mutex_unlock(&cdev_mutex);
576b5863 205
9ba682f0 206 log_debug(1 << CXGBI_DBG_DEV,
207 "lldev 0x%p, NO match found.\n", lldev);
208 return NULL;
209}
210EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev);
211
fc8d0590
AB		 212struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev,
213 int *port)
9ba682f0 214{
0b3d8947 215 struct net_device *vdev = NULL;
9ba682f0 216 struct cxgbi_device *cdev, *tmp;
217 int i;
218
0b3d8947 219 if (ndev->priv_flags & IFF_802_1Q_VLAN) {
220 vdev = ndev;
9ba682f0 221 ndev = vlan_dev_real_dev(ndev);
0b3d8947 222 log_debug(1 << CXGBI_DBG_DEV,
223 "vlan dev %s -> %s.\n", vdev->name, ndev->name);
224 }
9ba682f0 225
226 mutex_lock(&cdev_mutex);
227 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
228 for (i = 0; i < cdev->nports; i++) {
229 if (ndev == cdev->ports[i]) {
0b3d8947 230 cdev->hbas[i]->vdev = vdev;
9ba682f0 231 mutex_unlock(&cdev_mutex);
232 if (port)
233 *port = i;
234 return cdev;
235 }
236 }
237 }
238 mutex_unlock(&cdev_mutex);
239 log_debug(1 << CXGBI_DBG_DEV,
240 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
241 return NULL;
242}
fc8d0590
AB		 243EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev);
244
576b5863
AB		 245struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev,
246 int *port)
247{
248 struct net_device *vdev = NULL;
249 struct cxgbi_device *cdev;
250 int i;
251
252 if (ndev->priv_flags & IFF_802_1Q_VLAN) {
253 vdev = ndev;
254 ndev = vlan_dev_real_dev(ndev);
255 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
256 }
257
258 rcu_read_lock();
259 list_for_each_entry_rcu(cdev, &cdev_rcu_list, rcu_node) {
260 for (i = 0; i < cdev->nports; i++) {
261 if (ndev == cdev->ports[i]) {
262 cdev->hbas[i]->vdev = vdev;
263 rcu_read_unlock();
264 if (port)
265 *port = i;
266 return cdev;
267 }
268 }
269 }
270 rcu_read_unlock();
271
272 log_debug(1 << CXGBI_DBG_DEV,
273 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
274 return NULL;
275}
276EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu);
277
fc8d0590
AB		 278static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev,
279 int *port)
280{
281 struct net_device *vdev = NULL;
282 struct cxgbi_device *cdev, *tmp;
283 int i;
284
285 if (ndev->priv_flags & IFF_802_1Q_VLAN) {
286 vdev = ndev;
287 ndev = vlan_dev_real_dev(ndev);
288 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
289 }
290
291 mutex_lock(&cdev_mutex);
292 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
293 for (i = 0; i < cdev->nports; i++) {
294 if (!memcmp(ndev->dev_addr, cdev->ports[i]->dev_addr,
295 MAX_ADDR_LEN)) {
296 cdev->hbas[i]->vdev = vdev;
297 mutex_unlock(&cdev_mutex);
298 if (port)
299 *port = i;
300 return cdev;
301 }
302 }
303 }
304 mutex_unlock(&cdev_mutex);
305 log_debug(1 << CXGBI_DBG_DEV,
306 "ndev 0x%p, %s, NO match mac found.\n",
307 ndev, ndev->name);
308 return NULL;
309}
9ba682f0 310
9ba682f0 311void cxgbi_hbas_remove(struct cxgbi_device *cdev)
312{
313 int i;
314 struct cxgbi_hba *chba;
315
316 log_debug(1 << CXGBI_DBG_DEV,
317 "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
318
319 for (i = 0; i < cdev->nports; i++) {
320 chba = cdev->hbas[i];
321 if (chba) {
322 cdev->hbas[i] = NULL;
323 iscsi_host_remove(chba->shost);
324 pci_dev_put(cdev->pdev);
325 iscsi_host_free(chba->shost);
326 }
327 }
328}
329EXPORT_SYMBOL_GPL(cxgbi_hbas_remove);
330
1abf635d 331int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun,
9ba682f0 332 unsigned int max_id, struct scsi_host_template *sht,
333 struct scsi_transport_template *stt)
334{
335 struct cxgbi_hba *chba;
336 struct Scsi_Host *shost;
337 int i, err;
338
339 log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
340
341 for (i = 0; i < cdev->nports; i++) {
342 shost = iscsi_host_alloc(sht, sizeof(*chba), 1);
343 if (!shost) {
344 pr_info("0x%p, p%d, %s, host alloc failed.\n",
345 cdev, i, cdev->ports[i]->name);
346 err = -ENOMEM;
347 goto err_out;
348 }
349
350 shost->transportt = stt;
351 shost->max_lun = max_lun;
352 shost->max_id = max_id;
353 shost->max_channel = 0;
354 shost->max_cmd_len = 16;
355
356 chba = iscsi_host_priv(shost);
357 chba->cdev = cdev;
358 chba->ndev = cdev->ports[i];
359 chba->shost = shost;
360
361 log_debug(1 << CXGBI_DBG_DEV,
362 "cdev 0x%p, p#%d %s: chba 0x%p.\n",
363 cdev, i, cdev->ports[i]->name, chba);
364
365 pci_dev_get(cdev->pdev);
366 err = iscsi_host_add(shost, &cdev->pdev->dev);
367 if (err) {
368 pr_info("cdev 0x%p, p#%d %s, host add failed.\n",
369 cdev, i, cdev->ports[i]->name);
370 pci_dev_put(cdev->pdev);
371 scsi_host_put(shost);
372 goto err_out;
373 }
374
375 cdev->hbas[i] = chba;
376 }
377
378 return 0;
379
380err_out:
381 cxgbi_hbas_remove(cdev);
382 return err;
383}
384EXPORT_SYMBOL_GPL(cxgbi_hbas_add);
385
386/*
387 * iSCSI offload
388 *
389 * - source port management
390 * To find a free source port in the port allocation map we use a very simple
391 * rotor scheme to look for the next free port.
392 *
393 * If a source port has been specified make sure that it doesn't collide with
394 * our normal source port allocation map. If it's outside the range of our
395 * allocation/deallocation scheme just let them use it.
396 *
397 * If the source port is outside our allocation range, the caller is
398 * responsible for keeping track of their port usage.
399 */
400static int sock_get_port(struct cxgbi_sock *csk)
401{
402 struct cxgbi_device *cdev = csk->cdev;
403 struct cxgbi_ports_map *pmap = &cdev->pmap;
404 unsigned int start;
405 int idx;
fc8d0590 406 __be16 *port;
9ba682f0 407
408 if (!pmap->max_connect) {
409 pr_err("cdev 0x%p, p#%u %s, NO port map.\n",
410 cdev, csk->port_id, cdev->ports[csk->port_id]->name);
411 return -EADDRNOTAVAIL;
412 }
413
fc8d0590
AB		 414 if (csk->csk_family == AF_INET)
415 port = &csk->saddr.sin_port;
416 else /* ipv6 */
417 port = &csk->saddr6.sin6_port;
418
419 if (*port) {
9ba682f0 420 pr_err("source port NON-ZERO %u.\n",
fc8d0590 421 ntohs(*port));
9ba682f0 422 return -EADDRINUSE;
423 }
424
425 spin_lock_bh(&pmap->lock);
426 if (pmap->used >= pmap->max_connect) {
427 spin_unlock_bh(&pmap->lock);
428 pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n",
429 cdev, csk->port_id, cdev->ports[csk->port_id]->name);
430 return -EADDRNOTAVAIL;
431 }
432
433 start = idx = pmap->next;
434 do {
435 if (++idx >= pmap->max_connect)
436 idx = 0;
437 if (!pmap->port_csk[idx]) {
438 pmap->used++;
fc8d0590 439 *port = htons(pmap->sport_base + idx);
9ba682f0 440 pmap->next = idx;
441 pmap->port_csk[idx] = csk;
442 spin_unlock_bh(&pmap->lock);
443 cxgbi_sock_get(csk);
444 log_debug(1 << CXGBI_DBG_SOCK,
445 "cdev 0x%p, p#%u %s, p %u, %u.\n",
446 cdev, csk->port_id,
447 cdev->ports[csk->port_id]->name,
448 pmap->sport_base + idx, pmap->next);
449 return 0;
450 }
451 } while (idx != start);
452 spin_unlock_bh(&pmap->lock);
453
454 /* should not happen */
455 pr_warn("cdev 0x%p, p#%u %s, next %u?\n",
456 cdev, csk->port_id, cdev->ports[csk->port_id]->name,
457 pmap->next);
458 return -EADDRNOTAVAIL;
459}
460
461static void sock_put_port(struct cxgbi_sock *csk)
462{
463 struct cxgbi_device *cdev = csk->cdev;
464 struct cxgbi_ports_map *pmap = &cdev->pmap;
fc8d0590 465 __be16 *port;
9ba682f0 466
fc8d0590
AB		 467 if (csk->csk_family == AF_INET)
468 port = &csk->saddr.sin_port;
469 else /* ipv6 */
470 port = &csk->saddr6.sin6_port;
9ba682f0 471
fc8d0590
AB		 472 if (*port) {
473 int idx = ntohs(*port) - pmap->sport_base;
474
475 *port = 0;
9ba682f0 476 if (idx < 0 || idx >= pmap->max_connect) {
477 pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n",
478 cdev, csk->port_id,
479 cdev->ports[csk->port_id]->name,
fc8d0590 480 ntohs(*port));
9ba682f0 481 return;
482 }
483
484 spin_lock_bh(&pmap->lock);
485 pmap->port_csk[idx] = NULL;
486 pmap->used--;
487 spin_unlock_bh(&pmap->lock);
488
489 log_debug(1 << CXGBI_DBG_SOCK,
490 "cdev 0x%p, p#%u %s, release %u.\n",
491 cdev, csk->port_id, cdev->ports[csk->port_id]->name,
492 pmap->sport_base + idx);
493
494 cxgbi_sock_put(csk);
495 }
496}
497
498/*
499 * iscsi tcp connection
500 */
501void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk)
502{
503 if (csk->cpl_close) {
504 kfree_skb(csk->cpl_close);
505 csk->cpl_close = NULL;
506 }
507 if (csk->cpl_abort_req) {
508 kfree_skb(csk->cpl_abort_req);
509 csk->cpl_abort_req = NULL;
510 }
511 if (csk->cpl_abort_rpl) {
512 kfree_skb(csk->cpl_abort_rpl);
513 csk->cpl_abort_rpl = NULL;
514 }
515}
516EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs);
517
518static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev)
519{
520 struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO);
521
522 if (!csk) {
523 pr_info("alloc csk %zu failed.\n", sizeof(*csk));
524 return NULL;
525 }
526
527 if (cdev->csk_alloc_cpls(csk) < 0) {
528 pr_info("csk 0x%p, alloc cpls failed.\n", csk);
529 kfree(csk);
530 return NULL;
531 }
532
533 spin_lock_init(&csk->lock);
534 kref_init(&csk->refcnt);
535 skb_queue_head_init(&csk->receive_queue);
536 skb_queue_head_init(&csk->write_queue);
537 setup_timer(&csk->retry_timer, NULL, (unsigned long)csk);
538 rwlock_init(&csk->callback_lock);
539 csk->cdev = cdev;
540 csk->flags = 0;
541 cxgbi_sock_set_state(csk, CTP_CLOSED);
542
543 log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk);
544
545 return csk;
546}
547
f4bfd99f
DM		 548static struct rtable *find_route_ipv4(struct flowi4 *fl4,
549 __be32 saddr, __be32 daddr,
78fbfd8a 550 __be16 sport, __be16 dport, u8 tos)
9ba682f0 551{
552 struct rtable *rt;
9ba682f0 553
f4bfd99f 554 rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr,
78fbfd8a 555 dport, sport, IPPROTO_TCP, tos, 0);
b23dd4fe 556 if (IS_ERR(rt))
9ba682f0 557 return NULL;
558
559 return rt;
560}
561
562static struct cxgbi_sock *cxgbi_check_route(struct sockaddr *dst_addr)
563{
564 struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr;
565 struct dst_entry *dst;
566 struct net_device *ndev;
567 struct cxgbi_device *cdev;
568 struct rtable *rt = NULL;
a58b61e5 569 struct neighbour *n;
f4bfd99f 570 struct flowi4 fl4;
9ba682f0 571 struct cxgbi_sock *csk = NULL;
572 unsigned int mtu = 0;
573 int port = 0xFFFF;
574 int err = 0;
575
f4bfd99f 576 rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0, daddr->sin_port, 0);
9ba682f0 577 if (!rt) {
578 pr_info("no route to ipv4 0x%x, port %u.\n",
fc8d0590
AB		 579 be32_to_cpu(daddr->sin_addr.s_addr),
580 be16_to_cpu(daddr->sin_port));
9ba682f0 581 err = -ENETUNREACH;
582 goto err_out;
583 }
584 dst = &rt->dst;
0b399d46 585 n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr);
a58b61e5
DM		 586 if (!n) {
587 err = -ENODEV;
588 goto rel_rt;
589 }
590 ndev = n->dev;
9ba682f0 591
592 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
593 pr_info("multi-cast route %pI4, port %u, dev %s.\n",
594 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
595 ndev->name);
596 err = -ENETUNREACH;
0b399d46 597 goto rel_neigh;
9ba682f0 598 }
599
600 if (ndev->flags & IFF_LOOPBACK) {
601 ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr);
602 mtu = ndev->mtu;
603 pr_info("rt dev %s, loopback -> %s, mtu %u.\n",
a58b61e5 604 n->dev->name, ndev->name, mtu);
9ba682f0 605 }
606
9ba682f0 607 cdev = cxgbi_device_find_by_netdev(ndev, &port);
608 if (!cdev) {
609 pr_info("dst %pI4, %s, NOT cxgbi device.\n",
610 &daddr->sin_addr.s_addr, ndev->name);
611 err = -ENETUNREACH;
0b399d46 612 goto rel_neigh;
9ba682f0 613 }
614 log_debug(1 << CXGBI_DBG_SOCK,
615 "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n",
616 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
617 port, ndev->name, cdev);
618
619 csk = cxgbi_sock_create(cdev);
620 if (!csk) {
621 err = -ENOMEM;
0b399d46 622 goto rel_neigh;
9ba682f0 623 }
624 csk->cdev = cdev;
625 csk->port_id = port;
626 csk->mtu = mtu;
627 csk->dst = dst;
fc8d0590
AB		 628
629 csk->csk_family = AF_INET;
9ba682f0 630 csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr;
631 csk->daddr.sin_port = daddr->sin_port;
c71b9b66 632 csk->daddr.sin_family = daddr->sin_family;
fc8d0590 633 csk->saddr.sin_family = daddr->sin_family;
f4bfd99f 634 csk->saddr.sin_addr.s_addr = fl4.saddr;
0b399d46 635 neigh_release(n);
9ba682f0 636
637 return csk;
638
0b399d46
DM		 639rel_neigh:
640 neigh_release(n);
641
9ba682f0 642rel_rt:
643 ip_rt_put(rt);
644 if (csk)
645 cxgbi_sock_closed(csk);
646err_out:
647 return ERR_PTR(err);
648}
649
e81fbf6c 650#if IS_ENABLED(CONFIG_IPV6)
fc8d0590
AB		 651static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr,
652 const struct in6_addr *daddr)
653{
654 struct flowi6 fl;
655
656 if (saddr)
657 memcpy(&fl.saddr, saddr, sizeof(struct in6_addr));
658 if (daddr)
659 memcpy(&fl.daddr, daddr, sizeof(struct in6_addr));
660 return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl);
661}
662
663static struct cxgbi_sock *cxgbi_check_route6(struct sockaddr *dst_addr)
664{
665 struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr;
666 struct dst_entry *dst;
667 struct net_device *ndev;
668 struct cxgbi_device *cdev;
669 struct rt6_info *rt = NULL;
670 struct neighbour *n;
671 struct in6_addr pref_saddr;
672 struct cxgbi_sock *csk = NULL;
673 unsigned int mtu = 0;
674 int port = 0xFFFF;
675 int err = 0;
676
677 rt = find_route_ipv6(NULL, &daddr6->sin6_addr);
678
679 if (!rt) {
680 pr_info("no route to ipv6 %pI6 port %u\n",
681 daddr6->sin6_addr.s6_addr,
682 be16_to_cpu(daddr6->sin6_port));
683 err = -ENETUNREACH;
684 goto err_out;
685 }
686
687 dst = &rt->dst;
688
689 n = dst_neigh_lookup(dst, &daddr6->sin6_addr);
690
691 if (!n) {
692 pr_info("%pI6, port %u, dst no neighbour.\n",
693 daddr6->sin6_addr.s6_addr,
694 be16_to_cpu(daddr6->sin6_port));
695 err = -ENETUNREACH;
696 goto rel_rt;
697 }
698 ndev = n->dev;
699
700 if (ipv6_addr_is_multicast(&rt->rt6i_dst.addr)) {
701 pr_info("multi-cast route %pI6 port %u, dev %s.\n",
702 daddr6->sin6_addr.s6_addr,
703 ntohs(daddr6->sin6_port), ndev->name);
704 err = -ENETUNREACH;
705 goto rel_rt;
706 }
707
708 cdev = cxgbi_device_find_by_netdev(ndev, &port);
709 if (!cdev)
710 cdev = cxgbi_device_find_by_mac(ndev, &port);
711 if (!cdev) {
712 pr_info("dst %pI6 %s, NOT cxgbi device.\n",
713 daddr6->sin6_addr.s6_addr, ndev->name);
714 err = -ENETUNREACH;
715 goto rel_rt;
716 }
717 log_debug(1 << CXGBI_DBG_SOCK,
718 "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n",
719 daddr6->sin6_addr.s6_addr, ntohs(daddr6->sin6_port), port,
720 ndev->name, cdev);
721
722 csk = cxgbi_sock_create(cdev);
723 if (!csk) {
724 err = -ENOMEM;
725 goto rel_rt;
726 }
727 csk->cdev = cdev;
728 csk->port_id = port;
729 csk->mtu = mtu;
730 csk->dst = dst;
731
732 if (ipv6_addr_any(&rt->rt6i_prefsrc.addr)) {
733 struct inet6_dev *idev = ip6_dst_idev((struct dst_entry *)rt);
734
735 err = ipv6_dev_get_saddr(&init_net, idev ? idev->dev : NULL,
736 &daddr6->sin6_addr, 0, &pref_saddr);
737 if (err) {
738 pr_info("failed to get source address to reach %pI6\n",
739 &daddr6->sin6_addr);
740 goto rel_rt;
741 }
742 } else {
743 pref_saddr = rt->rt6i_prefsrc.addr;
744 }
745
746 csk->csk_family = AF_INET6;
747 csk->daddr6.sin6_addr = daddr6->sin6_addr;
748 csk->daddr6.sin6_port = daddr6->sin6_port;
749 csk->daddr6.sin6_family = daddr6->sin6_family;
750 csk->saddr6.sin6_addr = pref_saddr;
751
752 neigh_release(n);
753 return csk;
754
755rel_rt:
756 if (n)
757 neigh_release(n);
758
759 ip6_rt_put(rt);
760 if (csk)
761 cxgbi_sock_closed(csk);
762err_out:
763 return ERR_PTR(err);
764}
e81fbf6c 765#endif /* IS_ENABLED(CONFIG_IPV6) */
fc8d0590 766
9ba682f0 767void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn,
768 unsigned int opt)
769{
770 csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn;
771 dst_confirm(csk->dst);
772 smp_mb();
773 cxgbi_sock_set_state(csk, CTP_ESTABLISHED);
774}
775EXPORT_SYMBOL_GPL(cxgbi_sock_established);
776
777static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk)
778{
779 log_debug(1 << CXGBI_DBG_SOCK,
780 "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n",
781 csk, csk->state, csk->flags, csk->user_data);
782
783 if (csk->state != CTP_ESTABLISHED) {
e3d2ad8c 784 read_lock_bh(&csk->callback_lock);
9ba682f0 785 if (csk->user_data)
786 iscsi_conn_failure(csk->user_data,
787 ISCSI_ERR_CONN_FAILED);
e3d2ad8c 788 read_unlock_bh(&csk->callback_lock);
9ba682f0 789 }
790}
791
792void cxgbi_sock_closed(struct cxgbi_sock *csk)
793{
794 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
795 csk, (csk)->state, (csk)->flags, (csk)->tid);
796 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
797 if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED)
798 return;
799 if (csk->saddr.sin_port)
800 sock_put_port(csk);
801 if (csk->dst)
802 dst_release(csk->dst);
803 csk->cdev->csk_release_offload_resources(csk);
804 cxgbi_sock_set_state(csk, CTP_CLOSED);
805 cxgbi_inform_iscsi_conn_closing(csk);
806 cxgbi_sock_put(csk);
807}
808EXPORT_SYMBOL_GPL(cxgbi_sock_closed);
809
810static void need_active_close(struct cxgbi_sock *csk)
811{
812 int data_lost;
813 int close_req = 0;
814
815 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
816 csk, (csk)->state, (csk)->flags, (csk)->tid);
817 spin_lock_bh(&csk->lock);
818 dst_confirm(csk->dst);
819 data_lost = skb_queue_len(&csk->receive_queue);
820 __skb_queue_purge(&csk->receive_queue);
821
822 if (csk->state == CTP_ACTIVE_OPEN)
823 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
824 else if (csk->state == CTP_ESTABLISHED) {
825 close_req = 1;
826 cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE);
827 } else if (csk->state == CTP_PASSIVE_CLOSE) {
828 close_req = 1;
829 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
830 }
831
832 if (close_req) {
833 if (data_lost)
834 csk->cdev->csk_send_abort_req(csk);
835 else
836 csk->cdev->csk_send_close_req(csk);
837 }
838
839 spin_unlock_bh(&csk->lock);
840}
841
842void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno)
843{
844 pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n",
845 csk, csk->state, csk->flags,
846 &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port,
847 &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port,
848 errno);
849
850 cxgbi_sock_set_state(csk, CTP_CONNECTING);
851 csk->err = errno;
852 cxgbi_sock_closed(csk);
853}
854EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open);
855
856void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb)
857{
858 struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk;
859
860 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
861 csk, (csk)->state, (csk)->flags, (csk)->tid);
862 cxgbi_sock_get(csk);
863 spin_lock_bh(&csk->lock);
864 if (csk->state == CTP_ACTIVE_OPEN)
865 cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH);
866 spin_unlock_bh(&csk->lock);
867 cxgbi_sock_put(csk);
868 __kfree_skb(skb);
869}
870EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure);
871
872void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk)
873{
874 cxgbi_sock_get(csk);
875 spin_lock_bh(&csk->lock);
876 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
877 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_RCVD))
878 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD);
879 else {
880 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_RCVD);
881 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING);
882 if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD))
883 pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n",
884 csk, csk->state, csk->flags, csk->tid);
885 cxgbi_sock_closed(csk);
886 }
887 }
888 spin_unlock_bh(&csk->lock);
889 cxgbi_sock_put(csk);
890}
891EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl);
892
893void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk)
894{
895 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
896 csk, (csk)->state, (csk)->flags, (csk)->tid);
897 cxgbi_sock_get(csk);
898 spin_lock_bh(&csk->lock);
899
900 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
901 goto done;
902
903 switch (csk->state) {
904 case CTP_ESTABLISHED:
905 cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE);
906 break;
907 case CTP_ACTIVE_CLOSE:
908 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
909 break;
910 case CTP_CLOSE_WAIT_1:
911 cxgbi_sock_closed(csk);
912 break;
913 case CTP_ABORTING:
914 break;
915 default:
916 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
917 csk, csk->state, csk->flags, csk->tid);
918 }
919 cxgbi_inform_iscsi_conn_closing(csk);
920done:
921 spin_unlock_bh(&csk->lock);
922 cxgbi_sock_put(csk);
923}
924EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close);
925
926void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt)
927{
928 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
929 csk, (csk)->state, (csk)->flags, (csk)->tid);
930 cxgbi_sock_get(csk);
931 spin_lock_bh(&csk->lock);
932
933 csk->snd_una = snd_nxt - 1;
934 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
935 goto done;
936
937 switch (csk->state) {
938 case CTP_ACTIVE_CLOSE:
939 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1);
940 break;
941 case CTP_CLOSE_WAIT_1:
942 case CTP_CLOSE_WAIT_2:
943 cxgbi_sock_closed(csk);
944 break;
945 case CTP_ABORTING:
946 break;
947 default:
948 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
949 csk, csk->state, csk->flags, csk->tid);
950 }
951done:
952 spin_unlock_bh(&csk->lock);
953 cxgbi_sock_put(csk);
954}
955EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl);
956
957void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits,
958 unsigned int snd_una, int seq_chk)
959{
960 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
961 "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n",
962 csk, csk->state, csk->flags, csk->tid, credits,
963 csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk);
964
965 spin_lock_bh(&csk->lock);
966
967 csk->wr_cred += credits;
968 if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred)
969 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred;
970
971 while (credits) {
972 struct sk_buff *p = cxgbi_sock_peek_wr(csk);
973
974 if (unlikely(!p)) {
975 pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n",
976 csk, csk->state, csk->flags, csk->tid, credits,
977 csk->wr_cred, csk->wr_una_cred);
978 break;
979 }
980
981 if (unlikely(credits < p->csum)) {
982 pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n",
983 csk, csk->state, csk->flags, csk->tid,
984 credits, csk->wr_cred, csk->wr_una_cred,
985 p->csum);
986 p->csum -= credits;
987 break;
988 } else {
989 cxgbi_sock_dequeue_wr(csk);
990 credits -= p->csum;
991 kfree_skb(p);
992 }
993 }
994
995 cxgbi_sock_check_wr_invariants(csk);
996
997 if (seq_chk) {
998 if (unlikely(before(snd_una, csk->snd_una))) {
999 pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.",
1000 csk, csk->state, csk->flags, csk->tid, snd_una,
1001 csk->snd_una);
1002 goto done;
1003 }
1004
1005 if (csk->snd_una != snd_una) {
1006 csk->snd_una = snd_una;
1007 dst_confirm(csk->dst);
1008 }
1009 }
1010
1011 if (skb_queue_len(&csk->write_queue)) {
1012 if (csk->cdev->csk_push_tx_frames(csk, 0))
1013 cxgbi_conn_tx_open(csk);
1014 } else
1015 cxgbi_conn_tx_open(csk);
1016done:
1017 spin_unlock_bh(&csk->lock);
1018}
1019EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack);
1020
1021static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk,
1022 unsigned short mtu)
1023{
1024 int i = 0;
1025
1026 while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu)
1027 ++i;
1028
1029 return i;
1030}
1031
1032unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu)
1033{
1034 unsigned int idx;
1035 struct dst_entry *dst = csk->dst;
1036
0dbaee3b 1037 csk->advmss = dst_metric_advmss(dst);
9ba682f0 1038
1039 if (csk->advmss > pmtu - 40)
1040 csk->advmss = pmtu - 40;
1041 if (csk->advmss < csk->cdev->mtus[0] - 40)
1042 csk->advmss = csk->cdev->mtus[0] - 40;
1043 idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40);
1044
1045 return idx;
1046}
1047EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss);
1048
1049void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb)
1050{
1051 cxgbi_skcb_tcp_seq(skb) = csk->write_seq;
1052 __skb_queue_tail(&csk->write_queue, skb);
1053}
1054EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail);
1055
1056void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk)
1057{
1058 struct sk_buff *skb;
1059
1060 while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL)
1061 kfree_skb(skb);
1062}
1063EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue);
1064
1065void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk)
1066{
1067 int pending = cxgbi_sock_count_pending_wrs(csk);
1068
1069 if (unlikely(csk->wr_cred + pending != csk->wr_max_cred))
1070 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1071 csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred);
1072}
1073EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants);
1074
1075static int cxgbi_sock_send_pdus(struct cxgbi_sock *csk, struct sk_buff *skb)
1076{
1077 struct cxgbi_device *cdev = csk->cdev;
1078 struct sk_buff *next;
1079 int err, copied = 0;
1080
1081 spin_lock_bh(&csk->lock);
1082
1083 if (csk->state != CTP_ESTABLISHED) {
1084 log_debug(1 << CXGBI_DBG_PDU_TX,
1085 "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n",
1086 csk, csk->state, csk->flags, csk->tid);
1087 err = -EAGAIN;
1088 goto out_err;
1089 }
1090
1091 if (csk->err) {
1092 log_debug(1 << CXGBI_DBG_PDU_TX,
1093 "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n",
1094 csk, csk->state, csk->flags, csk->tid, csk->err);
1095 err = -EPIPE;
1096 goto out_err;
1097 }
1098
1099 if (csk->write_seq - csk->snd_una >= cdev->snd_win) {
1100 log_debug(1 << CXGBI_DBG_PDU_TX,
1101 "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n",
1102 csk, csk->state, csk->flags, csk->tid, csk->write_seq,
1103 csk->snd_una, cdev->snd_win);
1104 err = -ENOBUFS;
1105 goto out_err;
1106 }
1107
1108 while (skb) {
1109 int frags = skb_shinfo(skb)->nr_frags +
1110 (skb->len != skb->data_len);
1111
1112 if (unlikely(skb_headroom(skb) < cdev->skb_tx_rsvd)) {
1113 pr_err("csk 0x%p, skb head %u < %u.\n",
1114 csk, skb_headroom(skb), cdev->skb_tx_rsvd);
1115 err = -EINVAL;
1116 goto out_err;
1117 }
1118
1119 if (frags >= SKB_WR_LIST_SIZE) {
1120 pr_err("csk 0x%p, frags %d, %u,%u >%u.\n",
1121 csk, skb_shinfo(skb)->nr_frags, skb->len,
1122 skb->data_len, (uint)(SKB_WR_LIST_SIZE));
1123 err = -EINVAL;
1124 goto out_err;
1125 }
1126
1127 next = skb->next;
1128 skb->next = NULL;
1129 cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR);
1130 cxgbi_sock_skb_entail(csk, skb);
1131 copied += skb->len;
1132 csk->write_seq += skb->len +
1133 cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
1134 skb = next;
1135 }
1136done:
1137 if (likely(skb_queue_len(&csk->write_queue)))
1138 cdev->csk_push_tx_frames(csk, 1);
1139 spin_unlock_bh(&csk->lock);
1140 return copied;
1141
1142out_err:
1143 if (copied == 0 && err == -EPIPE)
1144 copied = csk->err ? csk->err : -EPIPE;
1145 else
1146 copied = err;
1147 goto done;
1148}
1149
1150/*
1151 * Direct Data Placement -
1152 * Directly place the iSCSI Data-In or Data-Out PDU's payload into pre-posted
1153 * final destination host-memory buffers based on the Initiator Task Tag (ITT)
1154 * in Data-In or Target Task Tag (TTT) in Data-Out PDUs.
1155 * The host memory address is programmed into h/w in the format of pagepod
1156 * entries.
1157 * The location of the pagepod entry is encoded into ddp tag which is used as
1158 * the base for ITT/TTT.
1159 */
1160
1161static unsigned char ddp_page_order[DDP_PGIDX_MAX] = {0, 1, 2, 4};
1162static unsigned char ddp_page_shift[DDP_PGIDX_MAX] = {12, 13, 14, 16};
1163static unsigned char page_idx = DDP_PGIDX_MAX;
1164
1165static unsigned char sw_tag_idx_bits;
1166static unsigned char sw_tag_age_bits;
1167
1168/*
1169 * Direct-Data Placement page size adjustment
1170 */
1171static int ddp_adjust_page_table(void)
1172{
1173 int i;
1174 unsigned int base_order, order;
1175
1176 if (PAGE_SIZE < (1UL << ddp_page_shift[0])) {
1177 pr_info("PAGE_SIZE 0x%lx too small, min 0x%lx\n",
1178 PAGE_SIZE, 1UL << ddp_page_shift[0]);
1179 return -EINVAL;
1180 }
1181
1182 base_order = get_order(1UL << ddp_page_shift[0]);
1183 order = get_order(1UL << PAGE_SHIFT);
1184
1185 for (i = 0; i < DDP_PGIDX_MAX; i++) {
1186 /* first is the kernel page size, then just doubling */
1187 ddp_page_order[i] = order - base_order + i;
1188 ddp_page_shift[i] = PAGE_SHIFT + i;
1189 }
1190 return 0;
1191}
1192
1193static int ddp_find_page_index(unsigned long pgsz)
1194{
1195 int i;
1196
1197 for (i = 0; i < DDP_PGIDX_MAX; i++) {
1198 if (pgsz == (1UL << ddp_page_shift[i]))
1199 return i;
1200 }
1201 pr_info("ddp page size %lu not supported.\n", pgsz);
1202 return DDP_PGIDX_MAX;
1203}
1204
1205static void ddp_setup_host_page_size(void)
1206{
1207 if (page_idx == DDP_PGIDX_MAX) {
1208 page_idx = ddp_find_page_index(PAGE_SIZE);
1209
1210 if (page_idx == DDP_PGIDX_MAX) {
1211 pr_info("system PAGE %lu, update hw.\n", PAGE_SIZE);
1212 if (ddp_adjust_page_table() < 0) {
1213 pr_info("PAGE %lu, disable ddp.\n", PAGE_SIZE);
1214 return;
1215 }
1216 page_idx = ddp_find_page_index(PAGE_SIZE);
1217 }
1218 pr_info("system PAGE %lu, ddp idx %u.\n", PAGE_SIZE, page_idx);
1219 }
1220}
1221
1222void cxgbi_ddp_page_size_factor(int *pgsz_factor)
1223{
1224 int i;
1225
1226 for (i = 0; i < DDP_PGIDX_MAX; i++)
1227 pgsz_factor[i] = ddp_page_order[i];
1228}
1229EXPORT_SYMBOL_GPL(cxgbi_ddp_page_size_factor);
1230
1231/*
1232 * DDP setup & teardown
1233 */
1234
1235void cxgbi_ddp_ppod_set(struct cxgbi_pagepod *ppod,
1236 struct cxgbi_pagepod_hdr *hdr,
1237 struct cxgbi_gather_list *gl, unsigned int gidx)
1238{
1239 int i;
1240
1241 memcpy(ppod, hdr, sizeof(*hdr));
1242 for (i = 0; i < (PPOD_PAGES_MAX + 1); i++, gidx++) {
1243 ppod->addr[i] = gidx < gl->nelem ?
1244 cpu_to_be64(gl->phys_addr[gidx]) : 0ULL;
1245 }
1246}
1247EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_set);
1248
1249void cxgbi_ddp_ppod_clear(struct cxgbi_pagepod *ppod)
1250{
1251 memset(ppod, 0, sizeof(*ppod));
1252}
1253EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_clear);
1254
1255static inline int ddp_find_unused_entries(struct cxgbi_ddp_info *ddp,
1256 unsigned int start, unsigned int max,
1257 unsigned int count,
1258 struct cxgbi_gather_list *gl)
1259{
1260 unsigned int i, j, k;
1261
1262 /* not enough entries */
1263 if ((max - start) < count) {
1264 log_debug(1 << CXGBI_DBG_DDP,
1265 "NOT enough entries %u+%u < %u.\n", start, count, max);
1266 return -EBUSY;
1267 }
1268
1269 max -= count;
1270 spin_lock(&ddp->map_lock);
1271 for (i = start; i < max;) {
1272 for (j = 0, k = i; j < count; j++, k++) {
1273 if (ddp->gl_map[k])
1274 break;
1275 }
1276 if (j == count) {
1277 for (j = 0, k = i; j < count; j++, k++)
1278 ddp->gl_map[k] = gl;
1279 spin_unlock(&ddp->map_lock);
1280 return i;
1281 }
1282 i += j + 1;
1283 }
1284 spin_unlock(&ddp->map_lock);
1285 log_debug(1 << CXGBI_DBG_DDP,
1286 "NO suitable entries %u available.\n", count);
1287 return -EBUSY;
1288}
1289
1290static inline void ddp_unmark_entries(struct cxgbi_ddp_info *ddp,
1291 int start, int count)
1292{
1293 spin_lock(&ddp->map_lock);
1294 memset(&ddp->gl_map[start], 0,
1295 count * sizeof(struct cxgbi_gather_list *));
1296 spin_unlock(&ddp->map_lock);
1297}
1298
1299static inline void ddp_gl_unmap(struct pci_dev *pdev,
1300 struct cxgbi_gather_list *gl)
1301{
1302 int i;
1303
1304 for (i = 0; i < gl->nelem; i++)
1305 dma_unmap_page(&pdev->dev, gl->phys_addr[i], PAGE_SIZE,
1306 PCI_DMA_FROMDEVICE);
1307}
1308
1309static inline int ddp_gl_map(struct pci_dev *pdev,
1310 struct cxgbi_gather_list *gl)
1311{
1312 int i;
1313
1314 for (i = 0; i < gl->nelem; i++) {
1315 gl->phys_addr[i] = dma_map_page(&pdev->dev, gl->pages[i], 0,
1316 PAGE_SIZE,
1317 PCI_DMA_FROMDEVICE);
1318 if (unlikely(dma_mapping_error(&pdev->dev, gl->phys_addr[i]))) {
1319 log_debug(1 << CXGBI_DBG_DDP,
1320 "page %d 0x%p, 0x%p dma mapping err.\n",
1321 i, gl->pages[i], pdev);
1322 goto unmap;
1323 }
1324 }
1325 return i;
1326unmap:
1327 if (i) {
1328 unsigned int nelem = gl->nelem;
1329
1330 gl->nelem = i;
1331 ddp_gl_unmap(pdev, gl);
1332 gl->nelem = nelem;
1333 }
1334 return -EINVAL;
1335}
1336
1337static void ddp_release_gl(struct cxgbi_gather_list *gl,
1338 struct pci_dev *pdev)
1339{
1340 ddp_gl_unmap(pdev, gl);
1341 kfree(gl);
1342}
1343
1344static struct cxgbi_gather_list *ddp_make_gl(unsigned int xferlen,
1345 struct scatterlist *sgl,
1346 unsigned int sgcnt,
1347 struct pci_dev *pdev,
1348 gfp_t gfp)
1349{
1350 struct cxgbi_gather_list *gl;
1351 struct scatterlist *sg = sgl;
1352 struct page *sgpage = sg_page(sg);
1353 unsigned int sglen = sg->length;
1354 unsigned int sgoffset = sg->offset;
1355 unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >>
1356 PAGE_SHIFT;
1357 int i = 1, j = 0;
1358
1359 if (xferlen < DDP_THRESHOLD) {
1360 log_debug(1 << CXGBI_DBG_DDP,
1361 "xfer %u < threshold %u, no ddp.\n",
1362 xferlen, DDP_THRESHOLD);
1363 return NULL;
1364 }
1365
1366 gl = kzalloc(sizeof(struct cxgbi_gather_list) +
1367 npages * (sizeof(dma_addr_t) +
1368 sizeof(struct page *)), gfp);
1369 if (!gl) {
1370 log_debug(1 << CXGBI_DBG_DDP,
1371 "xfer %u, %u pages, OOM.\n", xferlen, npages);
1372 return NULL;
1373 }
1374
1375 log_debug(1 << CXGBI_DBG_DDP,
1376 "xfer %u, sgl %u, gl max %u.\n", xferlen, sgcnt, npages);
1377
1378 gl->pages = (struct page **)&gl->phys_addr[npages];
1379 gl->nelem = npages;
1380 gl->length = xferlen;
1381 gl->offset = sgoffset;
1382 gl->pages[0] = sgpage;
1383
1384 for (i = 1, sg = sg_next(sgl), j = 0; i < sgcnt;
1385 i++, sg = sg_next(sg)) {
1386 struct page *page = sg_page(sg);
1387
1388 if (sgpage == page && sg->offset == sgoffset + sglen)
1389 sglen += sg->length;
1390 else {
1391 /* make sure the sgl is fit for ddp:
1392 * each has the same page size, and
1393 * all of the middle pages are used completely
1394 */
1395 if ((j && sgoffset) || ((i != sgcnt - 1) &&
1396 ((sglen + sgoffset) & ~PAGE_MASK))) {
1397 log_debug(1 << CXGBI_DBG_DDP,
1398 "page %d/%u, %u + %u.\n",
1399 i, sgcnt, sgoffset, sglen);
1400 goto error_out;
1401 }
1402
1403 j++;
1404 if (j == gl->nelem || sg->offset) {
1405 log_debug(1 << CXGBI_DBG_DDP,
1406 "page %d/%u, offset %u.\n",
1407 j, gl->nelem, sg->offset);
1408 goto error_out;
1409 }
1410 gl->pages[j] = page;
1411 sglen = sg->length;
1412 sgoffset = sg->offset;
1413 sgpage = page;
1414 }
1415 }
1416 gl->nelem = ++j;
1417
1418 if (ddp_gl_map(pdev, gl) < 0)
1419 goto error_out;
1420
1421 return gl;
1422
1423error_out:
1424 kfree(gl);
1425 return NULL;
1426}
1427
1428static void ddp_tag_release(struct cxgbi_hba *chba, u32 tag)
1429{
1430 struct cxgbi_device *cdev = chba->cdev;
1431 struct cxgbi_ddp_info *ddp = cdev->ddp;
1432 u32 idx;
1433
1434 idx = (tag >> PPOD_IDX_SHIFT) & ddp->idx_mask;
1435 if (idx < ddp->nppods) {
1436 struct cxgbi_gather_list *gl = ddp->gl_map[idx];
1437 unsigned int npods;
1438
1439 if (!gl || !gl->nelem) {
1440 pr_warn("tag 0x%x, idx %u, gl 0x%p, %u.\n",
1441 tag, idx, gl, gl ? gl->nelem : 0);
1442 return;
1443 }
1444 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
1445 log_debug(1 << CXGBI_DBG_DDP,
1446 "tag 0x%x, release idx %u, npods %u.\n",
1447 tag, idx, npods);
1448 cdev->csk_ddp_clear(chba, tag, idx, npods);
1449 ddp_unmark_entries(ddp, idx, npods);
1450 ddp_release_gl(gl, ddp->pdev);
1451 } else
1452 pr_warn("tag 0x%x, idx %u > max %u.\n", tag, idx, ddp->nppods);
1453}
1454
1455static int ddp_tag_reserve(struct cxgbi_sock *csk, unsigned int tid,
1456 u32 sw_tag, u32 *tagp, struct cxgbi_gather_list *gl,
1457 gfp_t gfp)
1458{
1459 struct cxgbi_device *cdev = csk->cdev;
1460 struct cxgbi_ddp_info *ddp = cdev->ddp;
1461 struct cxgbi_tag_format *tformat = &cdev->tag_format;
1462 struct cxgbi_pagepod_hdr hdr;
1463 unsigned int npods;
1464 int idx = -1;
1465 int err = -ENOMEM;
1466 u32 tag;
1467
1468 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
1469 if (ddp->idx_last == ddp->nppods)
1470 idx = ddp_find_unused_entries(ddp, 0, ddp->nppods,
1471 npods, gl);
1472 else {
1473 idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1,
1474 ddp->nppods, npods,
1475 gl);
1476 if (idx < 0 && ddp->idx_last >= npods) {
1477 idx = ddp_find_unused_entries(ddp, 0,
1478 min(ddp->idx_last + npods, ddp->nppods),
1479 npods, gl);
1480 }
1481 }
1482 if (idx < 0) {
1483 log_debug(1 << CXGBI_DBG_DDP,
1484 "xferlen %u, gl %u, npods %u NO DDP.\n",
1485 gl->length, gl->nelem, npods);
1486 return idx;
1487 }
1488
9ba682f0 1489 tag = cxgbi_ddp_tag_base(tformat, sw_tag);
1490 tag |= idx << PPOD_IDX_SHIFT;
1491
1492 hdr.rsvd = 0;
1493 hdr.vld_tid = htonl(PPOD_VALID_FLAG | PPOD_TID(tid));
1494 hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask);
1495 hdr.max_offset = htonl(gl->length);
1496 hdr.page_offset = htonl(gl->offset);
1497
1498 err = cdev->csk_ddp_set(csk, &hdr, idx, npods, gl);
b8ce8b59 1499 if (err < 0)
9ba682f0 1500 goto unmark_entries;
9ba682f0 1501
1502 ddp->idx_last = idx;
1503 log_debug(1 << CXGBI_DBG_DDP,
1504 "xfer %u, gl %u,%u, tid 0x%x, tag 0x%x->0x%x(%u,%u).\n",
1505 gl->length, gl->nelem, gl->offset, tid, sw_tag, tag, idx,
1506 npods);
1507 *tagp = tag;
1508 return 0;
1509
1510unmark_entries:
1511 ddp_unmark_entries(ddp, idx, npods);
1512 return err;
1513}
1514
1515int cxgbi_ddp_reserve(struct cxgbi_sock *csk, unsigned int *tagp,
1516 unsigned int sw_tag, unsigned int xferlen,
1517 struct scatterlist *sgl, unsigned int sgcnt, gfp_t gfp)
1518{
1519 struct cxgbi_device *cdev = csk->cdev;
1520 struct cxgbi_tag_format *tformat = &cdev->tag_format;
1521 struct cxgbi_gather_list *gl;
1522 int err;
1523
1524 if (page_idx >= DDP_PGIDX_MAX || !cdev->ddp ||
1525 xferlen < DDP_THRESHOLD) {
1526 log_debug(1 << CXGBI_DBG_DDP,
1527 "pgidx %u, xfer %u, NO ddp.\n", page_idx, xferlen);
1528 return -EINVAL;
1529 }
1530
1531 if (!cxgbi_sw_tag_usable(tformat, sw_tag)) {
1532 log_debug(1 << CXGBI_DBG_DDP,
1533 "sw_tag 0x%x NOT usable.\n", sw_tag);
1534 return -EINVAL;
1535 }
1536
1537 gl = ddp_make_gl(xferlen, sgl, sgcnt, cdev->pdev, gfp);
1538 if (!gl)
1539 return -ENOMEM;
1540
1541 err = ddp_tag_reserve(csk, csk->tid, sw_tag, tagp, gl, gfp);
1542 if (err < 0)
1543 ddp_release_gl(gl, cdev->pdev);
1544
1545 return err;
1546}
1547
1548static void ddp_destroy(struct kref *kref)
1549{
1550 struct cxgbi_ddp_info *ddp = container_of(kref,
1551 struct cxgbi_ddp_info,
1552 refcnt);
1553 struct cxgbi_device *cdev = ddp->cdev;
1554 int i = 0;
1555
1556 pr_info("kref 0, destroy ddp 0x%p, cdev 0x%p.\n", ddp, cdev);
1557
1558 while (i < ddp->nppods) {
1559 struct cxgbi_gather_list *gl = ddp->gl_map[i];
1560
1561 if (gl) {
1562 int npods = (gl->nelem + PPOD_PAGES_MAX - 1)
1563 >> PPOD_PAGES_SHIFT;
1564 pr_info("cdev 0x%p, ddp %d + %d.\n", cdev, i, npods);
1565 kfree(gl);
9ba682f0 1566 i += npods;
1567 } else
1568 i++;
1569 }
1570 cxgbi_free_big_mem(ddp);
1571}
1572
1573int cxgbi_ddp_cleanup(struct cxgbi_device *cdev)
1574{
1575 struct cxgbi_ddp_info *ddp = cdev->ddp;
1576
1577 log_debug(1 << CXGBI_DBG_DDP,
1578 "cdev 0x%p, release ddp 0x%p.\n", cdev, ddp);
1579 cdev->ddp = NULL;
1580 if (ddp)
1581 return kref_put(&ddp->refcnt, ddp_destroy);
1582 return 0;
1583}
1584EXPORT_SYMBOL_GPL(cxgbi_ddp_cleanup);
1585
1586int cxgbi_ddp_init(struct cxgbi_device *cdev,
1587 unsigned int llimit, unsigned int ulimit,
1588 unsigned int max_txsz, unsigned int max_rxsz)
1589{
1590 struct cxgbi_ddp_info *ddp;
1591 unsigned int ppmax, bits;
1592
1593 ppmax = (ulimit - llimit + 1) >> PPOD_SIZE_SHIFT;
1594 bits = __ilog2_u32(ppmax) + 1;
1595 if (bits > PPOD_IDX_MAX_SIZE)
1596 bits = PPOD_IDX_MAX_SIZE;
1597 ppmax = (1 << (bits - 1)) - 1;
1598
1599 ddp = cxgbi_alloc_big_mem(sizeof(struct cxgbi_ddp_info) +
1600 ppmax * (sizeof(struct cxgbi_gather_list *) +
1601 sizeof(struct sk_buff *)),
1602 GFP_KERNEL);
1603 if (!ddp) {
1604 pr_warn("cdev 0x%p, ddp ppmax %u OOM.\n", cdev, ppmax);
1605 return -ENOMEM;
1606 }
1607 ddp->gl_map = (struct cxgbi_gather_list **)(ddp + 1);
9ba682f0 1608 cdev->ddp = ddp;
1609
1610 spin_lock_init(&ddp->map_lock);
1611 kref_init(&ddp->refcnt);
1612
1613 ddp->cdev = cdev;
1614 ddp->pdev = cdev->pdev;
1615 ddp->llimit = llimit;
1616 ddp->ulimit = ulimit;
1617 ddp->max_txsz = min_t(unsigned int, max_txsz, ULP2_MAX_PKT_SIZE);
1618 ddp->max_rxsz = min_t(unsigned int, max_rxsz, ULP2_MAX_PKT_SIZE);
1619 ddp->nppods = ppmax;
1620 ddp->idx_last = ppmax;
1621 ddp->idx_bits = bits;
1622 ddp->idx_mask = (1 << bits) - 1;
1623 ddp->rsvd_tag_mask = (1 << (bits + PPOD_IDX_SHIFT)) - 1;
1624
1625 cdev->tag_format.sw_bits = sw_tag_idx_bits + sw_tag_age_bits;
1626 cdev->tag_format.rsvd_bits = ddp->idx_bits;
1627 cdev->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
1628 cdev->tag_format.rsvd_mask = (1 << cdev->tag_format.rsvd_bits) - 1;
1629
1630 pr_info("%s tag format, sw %u, rsvd %u,%u, mask 0x%x.\n",
1631 cdev->ports[0]->name, cdev->tag_format.sw_bits,
1632 cdev->tag_format.rsvd_bits, cdev->tag_format.rsvd_shift,
1633 cdev->tag_format.rsvd_mask);
1634
1635 cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1636 ddp->max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
1637 cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1638 ddp->max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
1639
1640 log_debug(1 << CXGBI_DBG_DDP,
1641 "%s max payload size: %u/%u, %u/%u.\n",
1642 cdev->ports[0]->name, cdev->tx_max_size, ddp->max_txsz,
1643 cdev->rx_max_size, ddp->max_rxsz);
1644 return 0;
1645}
1646EXPORT_SYMBOL_GPL(cxgbi_ddp_init);
1647
1648/*
1649 * APIs interacting with open-iscsi libraries
1650 */
1651
1652static unsigned char padding[4];
1653
1654static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt)
1655{
1656 struct scsi_cmnd *sc = task->sc;
1657 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
1658 struct cxgbi_conn *cconn = tcp_conn->dd_data;
1659 struct cxgbi_hba *chba = cconn->chba;
1660 struct cxgbi_tag_format *tformat = &chba->cdev->tag_format;
1661 u32 tag = ntohl((__force u32)hdr_itt);
1662
1663 log_debug(1 << CXGBI_DBG_DDP,
1664 "cdev 0x%p, release tag 0x%x.\n", chba->cdev, tag);
1665 if (sc &&
1666 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) &&
1667 cxgbi_is_ddp_tag(tformat, tag))
1668 ddp_tag_release(chba, tag);
1669}
1670
1671static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
1672{
1673 struct scsi_cmnd *sc = task->sc;
1674 struct iscsi_conn *conn = task->conn;
1675 struct iscsi_session *sess = conn->session;
1676 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1677 struct cxgbi_conn *cconn = tcp_conn->dd_data;
1678 struct cxgbi_hba *chba = cconn->chba;
1679 struct cxgbi_tag_format *tformat = &chba->cdev->tag_format;
1680 u32 sw_tag = (sess->age << cconn->task_idx_bits) | task->itt;
1681 u32 tag = 0;
1682 int err = -EINVAL;
1683
1684 if (sc &&
1685 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE)) {
1686 err = cxgbi_ddp_reserve(cconn->cep->csk, &tag, sw_tag,
1687 scsi_in(sc)->length,
1688 scsi_in(sc)->table.sgl,
1689 scsi_in(sc)->table.nents,
1690 GFP_ATOMIC);
1691 if (err < 0)
1692 log_debug(1 << CXGBI_DBG_DDP,
1693 "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n",
1694 cconn->cep->csk, task, scsi_in(sc)->length,
1695 scsi_in(sc)->table.nents);
1696 }
1697
1698 if (err < 0)
1699 tag = cxgbi_set_non_ddp_tag(tformat, sw_tag);
1700 /* the itt need to sent in big-endian order */
1701 *hdr_itt = (__force itt_t)htonl(tag);
1702
1703 log_debug(1 << CXGBI_DBG_DDP,
1704 "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n",
1705 chba->cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt);
1706 return 0;
1707}
1708
1709void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age)
1710{
1711 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1712 struct cxgbi_conn *cconn = tcp_conn->dd_data;
1713 struct cxgbi_device *cdev = cconn->chba->cdev;
1714 u32 tag = ntohl((__force u32) itt);
1715 u32 sw_bits;
1716
1717 sw_bits = cxgbi_tag_nonrsvd_bits(&cdev->tag_format, tag);
1718 if (idx)
1719 *idx = sw_bits & ((1 << cconn->task_idx_bits) - 1);
1720 if (age)
1721 *age = (sw_bits >> cconn->task_idx_bits) & ISCSI_AGE_MASK;
1722
1723 log_debug(1 << CXGBI_DBG_DDP,
1724 "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n",
1725 cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
1726 age ? *age : 0xFF);
1727}
1728EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt);
1729
1730void cxgbi_conn_tx_open(struct cxgbi_sock *csk)
1731{
1732 struct iscsi_conn *conn = csk->user_data;
1733
1734 if (conn) {
1735 log_debug(1 << CXGBI_DBG_SOCK,
1736 "csk 0x%p, cid %d.\n", csk, conn->id);
1737 iscsi_conn_queue_work(conn);
1738 }
1739}
1740EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open);
1741
1742/*
1743 * pdu receive, interact with libiscsi_tcp
1744 */
1745static inline int read_pdu_skb(struct iscsi_conn *conn,
1746 struct sk_buff *skb,
1747 unsigned int offset,
1748 int offloaded)
1749{
1750 int status = 0;
1751 int bytes_read;
1752
1753 bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
1754 switch (status) {
1755 case ISCSI_TCP_CONN_ERR:
1756 pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n",
1757 skb, offset, offloaded);
1758 return -EIO;
1759 case ISCSI_TCP_SUSPENDED:
1760 log_debug(1 << CXGBI_DBG_PDU_RX,
1761 "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n",
1762 skb, offset, offloaded, bytes_read);
1763 /* no transfer - just have caller flush queue */
1764 return bytes_read;
1765 case ISCSI_TCP_SKB_DONE:
1766 pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n",
1767 skb, offset, offloaded);
1768 /*
1769 * pdus should always fit in the skb and we should get
1770 * segment done notifcation.
1771 */
1772 iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
1773 return -EFAULT;
1774 case ISCSI_TCP_SEGMENT_DONE:
1775 log_debug(1 << CXGBI_DBG_PDU_RX,
1776 "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n",
1777 skb, offset, offloaded, bytes_read);
1778 return bytes_read;
1779 default:
1780 pr_info("skb 0x%p, off %u, %d, invalid status %d.\n",
1781 skb, offset, offloaded, status);
1782 return -EINVAL;
1783 }
1784}
1785
1786static int skb_read_pdu_bhs(struct iscsi_conn *conn, struct sk_buff *skb)
1787{
1788 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1789
1790 log_debug(1 << CXGBI_DBG_PDU_RX,
1791 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
1792 conn, skb, skb->len, cxgbi_skcb_flags(skb));
1793
1794 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
1795 pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb);
1796 iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
1797 return -EIO;
1798 }
1799
1800 if (conn->hdrdgst_en &&
1801 cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) {
1802 pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb);
1803 iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
1804 return -EIO;
1805 }
1806
1807 return read_pdu_skb(conn, skb, 0, 0);
1808}
1809
1810static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb,
1811 struct sk_buff *skb, unsigned int offset)
1812{
1813 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1814 bool offloaded = 0;
1815 int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
1816
1817 log_debug(1 << CXGBI_DBG_PDU_RX,
1818 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
1819 conn, skb, skb->len, cxgbi_skcb_flags(skb));
1820
1821 if (conn->datadgst_en &&
1822 cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) {
1823 pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n",
1824 conn, lskb, cxgbi_skcb_flags(lskb));
1825 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1826 return -EIO;
1827 }
1828
1829 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
1830 return 0;
1831
1832 /* coalesced, add header digest length */
1833 if (lskb == skb && conn->hdrdgst_en)
1834 offset += ISCSI_DIGEST_SIZE;
1835
1836 if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD))
1837 offloaded = 1;
1838
1839 if (opcode == ISCSI_OP_SCSI_DATA_IN)
1840 log_debug(1 << CXGBI_DBG_PDU_RX,
1841 "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n",
1842 skb, opcode, ntohl(tcp_conn->in.hdr->itt),
1843 tcp_conn->in.datalen, offloaded ? "is" : "not");
1844
1845 return read_pdu_skb(conn, skb, offset, offloaded);
1846}
1847
1848static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied)
1849{
1850 struct cxgbi_device *cdev = csk->cdev;
1851 int must_send;
1852 u32 credits;
1853
1854 log_debug(1 << CXGBI_DBG_PDU_RX,
fc385045 1855 "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n",
9ba682f0 1856 csk, csk->state, csk->flags, csk->tid, csk->copied_seq,
1857 csk->rcv_wup, cdev->rx_credit_thres,
1858 cdev->rcv_win);
1859
1860 if (csk->state != CTP_ESTABLISHED)
1861 return;
1862
1863 credits = csk->copied_seq - csk->rcv_wup;
1864 if (unlikely(!credits))
1865 return;
1866 if (unlikely(cdev->rx_credit_thres == 0))
1867 return;
1868
1869 must_send = credits + 16384 >= cdev->rcv_win;
1870 if (must_send || credits >= cdev->rx_credit_thres)
1871 csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits);
1872}
1873
1874void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk)
1875{
1876 struct cxgbi_device *cdev = csk->cdev;
1877 struct iscsi_conn *conn = csk->user_data;
1878 struct sk_buff *skb;
1879 unsigned int read = 0;
1880 int err = 0;
1881
1882 log_debug(1 << CXGBI_DBG_PDU_RX,
1883 "csk 0x%p, conn 0x%p.\n", csk, conn);
1884
1885 if (unlikely(!conn || conn->suspend_rx)) {
1886 log_debug(1 << CXGBI_DBG_PDU_RX,
1887 "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n",
1888 csk, conn, conn ? conn->id : 0xFF,
1889 conn ? conn->suspend_rx : 0xFF);
9ba682f0 1890 return;
1891 }
1892
1893 while (!err) {
9ba682f0 1894 skb = skb_peek(&csk->receive_queue);
1895 if (!skb ||
1896 !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) {
1897 if (skb)
1898 log_debug(1 << CXGBI_DBG_PDU_RX,
1899 "skb 0x%p, NOT ready 0x%lx.\n",
1900 skb, cxgbi_skcb_flags(skb));
9ba682f0 1901 break;
1902 }
1903 __skb_unlink(skb, &csk->receive_queue);
9ba682f0 1904
1905 read += cxgbi_skcb_rx_pdulen(skb);
1906 log_debug(1 << CXGBI_DBG_PDU_RX,
1907 "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n",
1908 csk, skb, skb->len, cxgbi_skcb_flags(skb),
1909 cxgbi_skcb_rx_pdulen(skb));
1910
1911 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) {
1912 err = skb_read_pdu_bhs(conn, skb);
e3d2ad8c 1913 if (err < 0) {
1914 pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, "
1915 "f 0x%lx, plen %u.\n",
1916 csk, skb, skb->len,
1917 cxgbi_skcb_flags(skb),
1918 cxgbi_skcb_rx_pdulen(skb));
1919 goto skb_done;
1920 }
9ba682f0 1921 err = skb_read_pdu_data(conn, skb, skb,
1922 err + cdev->skb_rx_extra);
e3d2ad8c 1923 if (err < 0)
1924 pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, "
1925 "f 0x%lx, plen %u.\n",
1926 csk, skb, skb->len,
1927 cxgbi_skcb_flags(skb),
1928 cxgbi_skcb_rx_pdulen(skb));
9ba682f0 1929 } else {
1930 err = skb_read_pdu_bhs(conn, skb);
e3d2ad8c 1931 if (err < 0) {
1932 pr_err("bhs, csk 0x%p, skb 0x%p,%u, "
1933 "f 0x%lx, plen %u.\n",
1934 csk, skb, skb->len,
1935 cxgbi_skcb_flags(skb),
1936 cxgbi_skcb_rx_pdulen(skb));
1937 goto skb_done;
1938 }
1939
9ba682f0 1940 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) {
1941 struct sk_buff *dskb;
1942
9ba682f0 1943 dskb = skb_peek(&csk->receive_queue);
1944 if (!dskb) {
e3d2ad8c 1945 pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx,"
1946 " plen %u, NO data.\n",
1947 csk, skb, skb->len,
1948 cxgbi_skcb_flags(skb),
1949 cxgbi_skcb_rx_pdulen(skb));
1950 err = -EIO;
1951 goto skb_done;
9ba682f0 1952 }
1953 __skb_unlink(dskb, &csk->receive_queue);
9ba682f0 1954
1955 err = skb_read_pdu_data(conn, skb, dskb, 0);
e3d2ad8c 1956 if (err < 0)
1957 pr_err("data, csk 0x%p, skb 0x%p,%u, "
1958 "f 0x%lx, plen %u, dskb 0x%p,"
1959 "%u.\n",
1960 csk, skb, skb->len,
1961 cxgbi_skcb_flags(skb),
1962 cxgbi_skcb_rx_pdulen(skb),
1963 dskb, dskb->len);
9ba682f0 1964 __kfree_skb(dskb);
1965 } else
1966 err = skb_read_pdu_data(conn, skb, skb, 0);
1967 }
e3d2ad8c 1968skb_done:
1969 __kfree_skb(skb);
1970
9ba682f0 1971 if (err < 0)
1972 break;
9ba682f0 1973 }
1974
1975 log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read);
1976 if (read) {
1977 csk->copied_seq += read;
1978 csk_return_rx_credits(csk, read);
1979 conn->rxdata_octets += read;
1980 }
1981
1982 if (err < 0) {
e3d2ad8c 1983 pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n",
1984 csk, conn, err, read);
9ba682f0 1985 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1986 }
1987}
1988EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready);
1989
1990static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt,
1991 unsigned int offset, unsigned int *off,
1992 struct scatterlist **sgp)
1993{
1994 int i;
1995 struct scatterlist *sg;
1996
1997 for_each_sg(sgl, sg, sgcnt, i) {
1998 if (offset < sg->length) {
1999 *off = offset;
2000 *sgp = sg;
2001 return 0;
2002 }
2003 offset -= sg->length;
2004 }
2005 return -EFAULT;
2006}
2007
2008static int sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
6a39a16a 2009 unsigned int dlen, struct page_frag *frags,
9ba682f0 2010 int frag_max)
2011{
2012 unsigned int datalen = dlen;
2013 unsigned int sglen = sg->length - sgoffset;
2014 struct page *page = sg_page(sg);
2015 int i;
2016
2017 i = 0;
2018 do {
2019 unsigned int copy;
2020
2021 if (!sglen) {
2022 sg = sg_next(sg);
2023 if (!sg) {
2024 pr_warn("sg %d NULL, len %u/%u.\n",
2025 i, datalen, dlen);
2026 return -EINVAL;
2027 }
2028 sgoffset = 0;
2029 sglen = sg->length;
2030 page = sg_page(sg);
2031
2032 }
2033 copy = min(datalen, sglen);
2034 if (i && page == frags[i - 1].page &&
2035 sgoffset + sg->offset ==
6a39a16a
IC		 2036 frags[i - 1].offset + frags[i - 1].size) {
2037 frags[i - 1].size += copy;
9ba682f0 2038 } else {
2039 if (i >= frag_max) {
2040 pr_warn("too many pages %u, dlen %u.\n",
2041 frag_max, dlen);
2042 return -EINVAL;
2043 }
2044
2045 frags[i].page = page;
6a39a16a
IC		 2046 frags[i].offset = sg->offset + sgoffset;
2047 frags[i].size = copy;
9ba682f0 2048 i++;
2049 }
2050 datalen -= copy;
2051 sgoffset += copy;
2052 sglen -= copy;
2053 } while (datalen);
2054
2055 return i;
2056}
2057
2058int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
2059{
2060 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
2061 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2062 struct cxgbi_device *cdev = cconn->chba->cdev;
2063 struct iscsi_conn *conn = task->conn;
2064 struct iscsi_tcp_task *tcp_task = task->dd_data;
e3d2ad8c 2065 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
9ba682f0 2066 struct scsi_cmnd *sc = task->sc;
2067 int headroom = SKB_TX_ISCSI_PDU_HEADER_MAX;
2068
2069 tcp_task->dd_data = tdata;
2070 task->hdr = NULL;
2071
2072 if (SKB_MAX_HEAD(cdev->skb_tx_rsvd) > (512 * MAX_SKB_FRAGS) &&
2073 (opcode == ISCSI_OP_SCSI_DATA_OUT ||
2074 (opcode == ISCSI_OP_SCSI_CMD &&
2075 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_TO_DEVICE))))
2076 /* data could goes into skb head */
2077 headroom += min_t(unsigned int,
2078 SKB_MAX_HEAD(cdev->skb_tx_rsvd),
2079 conn->max_xmit_dlength);
2080
2081 tdata->skb = alloc_skb(cdev->skb_tx_rsvd + headroom, GFP_ATOMIC);
2082 if (!tdata->skb) {
00c4a09b
TLSC		 2083 struct cxgbi_sock *csk = cconn->cep->csk;
2084 struct net_device *ndev = cdev->ports[csk->port_id];
2085 ndev->stats.tx_dropped++;
9ba682f0 2086 return -ENOMEM;
2087 }
2088
2089 skb_reserve(tdata->skb, cdev->skb_tx_rsvd);
2090 task->hdr = (struct iscsi_hdr *)tdata->skb->data;
2091 task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX; /* BHS + AHS */
2092
2093 /* data_out uses scsi_cmd's itt */
2094 if (opcode != ISCSI_OP_SCSI_DATA_OUT)
2095 task_reserve_itt(task, &task->hdr->itt);
2096
2097 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2098 "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n",
2099 task, opcode, tdata->skb, cdev->skb_tx_rsvd, headroom,
2100 conn->max_xmit_dlength, ntohl(task->hdr->itt));
2101
2102 return 0;
2103}
2104EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu);
2105
2106static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
2107{
c343a01c 2108 if (hcrc || dcrc) {
2109 u8 submode = 0;
9ba682f0 2110
c343a01c 2111 if (hcrc)
2112 submode |= 1;
2113 if (dcrc)
2114 submode |= 2;
2115 cxgbi_skcb_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode;
2116 } else
2117 cxgbi_skcb_ulp_mode(skb) = 0;
9ba682f0 2118}
2119
2120int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
2121 unsigned int count)
2122{
2123 struct iscsi_conn *conn = task->conn;
e3d2ad8c 2124 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
9ba682f0 2125 struct sk_buff *skb = tdata->skb;
2126 unsigned int datalen = count;
2127 int i, padlen = iscsi_padding(count);
2128 struct page *pg;
2129
2130 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2131 "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n",
2132 task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK,
2133 ntohl(task->cmdsn), ntohl(task->hdr->itt), offset, count);
2134
2135 skb_put(skb, task->hdr_len);
2136 tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
2137 if (!count)
2138 return 0;
2139
2140 if (task->sc) {
2141 struct scsi_data_buffer *sdb = scsi_out(task->sc);
2142 struct scatterlist *sg = NULL;
2143 int err;
2144
2145 tdata->offset = offset;
2146 tdata->count = count;
2147 err = sgl_seek_offset(
2148 sdb->table.sgl, sdb->table.nents,
2149 tdata->offset, &tdata->sgoffset, &sg);
2150 if (err < 0) {
2151 pr_warn("tpdu, sgl %u, bad offset %u/%u.\n",
2152 sdb->table.nents, tdata->offset, sdb->length);
2153 return err;
2154 }
2155 err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count,
2156 tdata->frags, MAX_PDU_FRAGS);
2157 if (err < 0) {
2158 pr_warn("tpdu, sgl %u, bad offset %u + %u.\n",
2159 sdb->table.nents, tdata->offset, tdata->count);
2160 return err;
2161 }
2162 tdata->nr_frags = err;
2163
2164 if (tdata->nr_frags > MAX_SKB_FRAGS ||
2165 (padlen && tdata->nr_frags == MAX_SKB_FRAGS)) {
2166 char *dst = skb->data + task->hdr_len;
6a39a16a 2167 struct page_frag *frag = tdata->frags;
9ba682f0 2168
2169 /* data fits in the skb's headroom */
2170 for (i = 0; i < tdata->nr_frags; i++, frag++) {
77dfce07 2171 char *src = kmap_atomic(frag->page);
9ba682f0 2172
6a39a16a
IC		 2173 memcpy(dst, src+frag->offset, frag->size);
2174 dst += frag->size;
77dfce07 2175 kunmap_atomic(src);
9ba682f0 2176 }
2177 if (padlen) {
2178 memset(dst, 0, padlen);
2179 padlen = 0;
2180 }
2181 skb_put(skb, count + padlen);
2182 } else {
2183 /* data fit into frag_list */
6a39a16a
IC		 2184 for (i = 0; i < tdata->nr_frags; i++) {
2185 __skb_fill_page_desc(skb, i,
2186 tdata->frags[i].page,
2187 tdata->frags[i].offset,
2188 tdata->frags[i].size);
2189 skb_frag_ref(skb, i);
2190 }
9ba682f0 2191 skb_shinfo(skb)->nr_frags = tdata->nr_frags;
2192 skb->len += count;
2193 skb->data_len += count;
2194 skb->truesize += count;
2195 }
2196
2197 } else {
2198 pg = virt_to_page(task->data);
2199
2200 get_page(pg);
2201 skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data),
2202 count);
2203 skb->len += count;
2204 skb->data_len += count;
2205 skb->truesize += count;
2206 }
2207
2208 if (padlen) {
2209 i = skb_shinfo(skb)->nr_frags;
2210 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
2211 virt_to_page(padding), offset_in_page(padding),
2212 padlen);
2213
2214 skb->data_len += padlen;
2215 skb->truesize += padlen;
2216 skb->len += padlen;
2217 }
2218
2219 return 0;
2220}
2221EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu);
2222
2223int cxgbi_conn_xmit_pdu(struct iscsi_task *task)
2224{
2225 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
2226 struct cxgbi_conn *cconn = tcp_conn->dd_data;
e3d2ad8c 2227 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
9ba682f0 2228 struct sk_buff *skb = tdata->skb;
2229 unsigned int datalen;
2230 int err;
2231
2232 if (!skb) {
2233 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2234 "task 0x%p, skb NULL.\n", task);
2235 return 0;
2236 }
2237
2238 datalen = skb->data_len;
2239 tdata->skb = NULL;
2240 err = cxgbi_sock_send_pdus(cconn->cep->csk, skb);
2241 if (err > 0) {
2242 int pdulen = err;
2243
2244 log_debug(1 << CXGBI_DBG_PDU_TX,
2245 "task 0x%p,0x%p, skb 0x%p, len %u/%u, rv %d.\n",
2246 task, task->sc, skb, skb->len, skb->data_len, err);
2247
2248 if (task->conn->hdrdgst_en)
2249 pdulen += ISCSI_DIGEST_SIZE;
2250
2251 if (datalen && task->conn->datadgst_en)
2252 pdulen += ISCSI_DIGEST_SIZE;
2253
2254 task->conn->txdata_octets += pdulen;
2255 return 0;
2256 }
2257
2258 if (err == -EAGAIN || err == -ENOBUFS) {
2259 log_debug(1 << CXGBI_DBG_PDU_TX,
2260 "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n",
2261 task, skb, skb->len, skb->data_len, err);
2262 /* reset skb to send when we are called again */
2263 tdata->skb = skb;
2264 return err;
2265 }
2266
2267 kfree_skb(skb);
2268 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2269 "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
2270 task->itt, skb, skb->len, skb->data_len, err);
2271 iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
2272 iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
2273 return err;
2274}
2275EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu);
2276
2277void cxgbi_cleanup_task(struct iscsi_task *task)
2278{
e3d2ad8c 2279 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
9ba682f0 2280
2281 log_debug(1 << CXGBI_DBG_ISCSI,
2282 "task 0x%p, skb 0x%p, itt 0x%x.\n",
2283 task, tdata->skb, task->hdr_itt);
2284
2285 /* never reached the xmit task callout */
2286 if (tdata->skb)
2287 __kfree_skb(tdata->skb);
2288 memset(tdata, 0, sizeof(*tdata));
2289
2290 task_release_itt(task, task->hdr_itt);
2291 iscsi_tcp_cleanup_task(task);
2292}
2293EXPORT_SYMBOL_GPL(cxgbi_cleanup_task);
2294
2295void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn,
2296 struct iscsi_stats *stats)
2297{
2298 struct iscsi_conn *conn = cls_conn->dd_data;
2299
2300 stats->txdata_octets = conn->txdata_octets;
2301 stats->rxdata_octets = conn->rxdata_octets;
2302 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
2303 stats->dataout_pdus = conn->dataout_pdus_cnt;
2304 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
2305 stats->datain_pdus = conn->datain_pdus_cnt;
2306 stats->r2t_pdus = conn->r2t_pdus_cnt;
2307 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
2308 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
2309 stats->digest_err = 0;
2310 stats->timeout_err = 0;
2311 stats->custom_length = 1;
2312 strcpy(stats->custom[0].desc, "eh_abort_cnt");
2313 stats->custom[0].value = conn->eh_abort_cnt;
2314}
2315EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats);
2316
2317static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn)
2318{
2319 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2320 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2321 struct cxgbi_device *cdev = cconn->chba->cdev;
2322 unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd);
2323 unsigned int max_def = 512 * MAX_SKB_FRAGS;
2324 unsigned int max = max(max_def, headroom);
2325
2326 max = min(cconn->chba->cdev->tx_max_size, max);
2327 if (conn->max_xmit_dlength)
2328 conn->max_xmit_dlength = min(conn->max_xmit_dlength, max);
2329 else
2330 conn->max_xmit_dlength = max;
2331 cxgbi_align_pdu_size(conn->max_xmit_dlength);
2332
2333 return 0;
2334}
2335
2336static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn)
2337{
2338 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2339 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2340 unsigned int max = cconn->chba->cdev->rx_max_size;
2341
2342 cxgbi_align_pdu_size(max);
2343
2344 if (conn->max_recv_dlength) {
2345 if (conn->max_recv_dlength > max) {
2346 pr_err("MaxRecvDataSegmentLength %u > %u.\n",
2347 conn->max_recv_dlength, max);
2348 return -EINVAL;
2349 }
2350 conn->max_recv_dlength = min(conn->max_recv_dlength, max);
2351 cxgbi_align_pdu_size(conn->max_recv_dlength);
2352 } else
2353 conn->max_recv_dlength = max;
2354
2355 return 0;
2356}
2357
2358int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn,
2359 enum iscsi_param param, char *buf, int buflen)
2360{
2361 struct iscsi_conn *conn = cls_conn->dd_data;
9ba682f0 2362 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2363 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2364 struct cxgbi_sock *csk = cconn->cep->csk;
1304be5f 2365 int err;
9ba682f0 2366
2367 log_debug(1 << CXGBI_DBG_ISCSI,
2368 "cls_conn 0x%p, param %d, buf(%d) %s.\n",
2369 cls_conn, param, buflen, buf);
2370
2371 switch (param) {
2372 case ISCSI_PARAM_HDRDGST_EN:
2373 err = iscsi_set_param(cls_conn, param, buf, buflen);
2374 if (!err && conn->hdrdgst_en)
2375 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
2376 conn->hdrdgst_en,
2377 conn->datadgst_en, 0);
2378 break;
2379 case ISCSI_PARAM_DATADGST_EN:
2380 err = iscsi_set_param(cls_conn, param, buf, buflen);
2381 if (!err && conn->datadgst_en)
2382 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
2383 conn->hdrdgst_en,
2384 conn->datadgst_en, 0);
2385 break;
2386 case ISCSI_PARAM_MAX_R2T:
1304be5f 2387 return iscsi_tcp_set_max_r2t(conn, buf);
9ba682f0 2388 case ISCSI_PARAM_MAX_RECV_DLENGTH:
2389 err = iscsi_set_param(cls_conn, param, buf, buflen);
2390 if (!err)
2391 err = cxgbi_conn_max_recv_dlength(conn);
2392 break;
2393 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2394 err = iscsi_set_param(cls_conn, param, buf, buflen);
2395 if (!err)
2396 err = cxgbi_conn_max_xmit_dlength(conn);
2397 break;
2398 default:
2399 return iscsi_set_param(cls_conn, param, buf, buflen);
2400 }
2401 return err;
2402}
2403EXPORT_SYMBOL_GPL(cxgbi_set_conn_param);
2404
fc8d0590
AB		 2405static inline int csk_print_port(struct cxgbi_sock *csk, char *buf)
2406{
2407 int len;
2408
2409 cxgbi_sock_get(csk);
2410 len = sprintf(buf, "%hu\n", ntohs(csk->daddr.sin_port));
2411 cxgbi_sock_put(csk);
2412
2413 return len;
2414}
2415
2416static inline int csk_print_ip(struct cxgbi_sock *csk, char *buf)
2417{
2418 int len;
2419
2420 cxgbi_sock_get(csk);
2421 if (csk->csk_family == AF_INET)
2422 len = sprintf(buf, "%pI4",
2423 &csk->daddr.sin_addr.s_addr);
2424 else
2425 len = sprintf(buf, "%pI6",
2426 &csk->daddr6.sin6_addr);
2427
2428 cxgbi_sock_put(csk);
2429
2430 return len;
2431}
2432
c71b9b66
MC		 2433int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param,
2434 char *buf)
9ba682f0 2435{
c71b9b66
MC		 2436 struct cxgbi_endpoint *cep = ep->dd_data;
2437 struct cxgbi_sock *csk;
9ba682f0 2438 int len;
2439
2440 log_debug(1 << CXGBI_DBG_ISCSI,
c71b9b66 2441 "cls_conn 0x%p, param %d.\n", ep, param);
9ba682f0 2442
2443 switch (param) {
2444 case ISCSI_PARAM_CONN_PORT:
9ba682f0 2445 case ISCSI_PARAM_CONN_ADDRESS:
c71b9b66
MC		 2446 if (!cep)
2447 return -ENOTCONN;
2448
2449 csk = cep->csk;
2450 if (!csk)
2451 return -ENOTCONN;
2452
2453 return iscsi_conn_get_addr_param((struct sockaddr_storage *)
2454 &csk->daddr, param, buf);
9ba682f0 2455 default:
c71b9b66 2456 return -ENOSYS;
9ba682f0 2457 }
2458 return len;
2459}
c71b9b66 2460EXPORT_SYMBOL_GPL(cxgbi_get_ep_param);
9ba682f0 2461
2462struct iscsi_cls_conn *
2463cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid)
2464{
2465 struct iscsi_cls_conn *cls_conn;
2466 struct iscsi_conn *conn;
2467 struct iscsi_tcp_conn *tcp_conn;
2468 struct cxgbi_conn *cconn;
2469
2470 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid);
2471 if (!cls_conn)
2472 return NULL;
2473
2474 conn = cls_conn->dd_data;
2475 tcp_conn = conn->dd_data;
2476 cconn = tcp_conn->dd_data;
2477 cconn->iconn = conn;
2478
2479 log_debug(1 << CXGBI_DBG_ISCSI,
2480 "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n",
2481 cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn);
2482
2483 return cls_conn;
2484}
2485EXPORT_SYMBOL_GPL(cxgbi_create_conn);
2486
2487int cxgbi_bind_conn(struct iscsi_cls_session *cls_session,
2488 struct iscsi_cls_conn *cls_conn,
2489 u64 transport_eph, int is_leading)
2490{
2491 struct iscsi_conn *conn = cls_conn->dd_data;
2492 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2493 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2494 struct iscsi_endpoint *ep;
2495 struct cxgbi_endpoint *cep;
2496 struct cxgbi_sock *csk;
2497 int err;
2498
2499 ep = iscsi_lookup_endpoint(transport_eph);
2500 if (!ep)
2501 return -EINVAL;
2502
2503 /* setup ddp pagesize */
2504 cep = ep->dd_data;
2505 csk = cep->csk;
2506 err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid, page_idx, 0);
2507 if (err < 0)
2508 return err;
2509
2510 err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
2511 if (err)
2512 return -EINVAL;
2513
2514 /* calculate the tag idx bits needed for this conn based on cmds_max */
2515 cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1;
2516
e3d2ad8c 2517 write_lock_bh(&csk->callback_lock);
9ba682f0 2518 csk->user_data = conn;
2519 cconn->chba = cep->chba;
2520 cconn->cep = cep;
2521 cep->cconn = cconn;
e3d2ad8c 2522 write_unlock_bh(&csk->callback_lock);
9ba682f0 2523
2524 cxgbi_conn_max_xmit_dlength(conn);
2525 cxgbi_conn_max_recv_dlength(conn);
2526
9ba682f0 2527 log_debug(1 << CXGBI_DBG_ISCSI,
2528 "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n",
2529 cls_session, cls_conn, ep, cconn, csk);
2530 /* init recv engine */
2531 iscsi_tcp_hdr_recv_prep(tcp_conn);
2532
2533 return 0;
2534}
2535EXPORT_SYMBOL_GPL(cxgbi_bind_conn);
2536
2537struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep,
2538 u16 cmds_max, u16 qdepth,
2539 u32 initial_cmdsn)
2540{
2541 struct cxgbi_endpoint *cep;
2542 struct cxgbi_hba *chba;
2543 struct Scsi_Host *shost;
2544 struct iscsi_cls_session *cls_session;
2545 struct iscsi_session *session;
2546
2547 if (!ep) {
2548 pr_err("missing endpoint.\n");
2549 return NULL;
2550 }
2551
2552 cep = ep->dd_data;
2553 chba = cep->chba;
2554 shost = chba->shost;
2555
2556 BUG_ON(chba != iscsi_host_priv(shost));
2557
2558 cls_session = iscsi_session_setup(chba->cdev->itp, shost,
2559 cmds_max, 0,
2560 sizeof(struct iscsi_tcp_task) +
2561 sizeof(struct cxgbi_task_data),
2562 initial_cmdsn, ISCSI_MAX_TARGET);
2563 if (!cls_session)
2564 return NULL;
2565
2566 session = cls_session->dd_data;
2567 if (iscsi_tcp_r2tpool_alloc(session))
2568 goto remove_session;
2569
2570 log_debug(1 << CXGBI_DBG_ISCSI,
2571 "ep 0x%p, cls sess 0x%p.\n", ep, cls_session);
2572 return cls_session;
2573
2574remove_session:
2575 iscsi_session_teardown(cls_session);
2576 return NULL;
2577}
2578EXPORT_SYMBOL_GPL(cxgbi_create_session);
2579
2580void cxgbi_destroy_session(struct iscsi_cls_session *cls_session)
2581{
2582 log_debug(1 << CXGBI_DBG_ISCSI,
2583 "cls sess 0x%p.\n", cls_session);
2584
2585 iscsi_tcp_r2tpool_free(cls_session->dd_data);
2586 iscsi_session_teardown(cls_session);
2587}
2588EXPORT_SYMBOL_GPL(cxgbi_destroy_session);
2589
2590int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
2591 char *buf, int buflen)
2592{
2593 struct cxgbi_hba *chba = iscsi_host_priv(shost);
2594
2595 if (!chba->ndev) {
2596 shost_printk(KERN_ERR, shost, "Could not get host param. "
2597 "netdev for host not set.\n");
2598 return -ENODEV;
2599 }
2600
2601 log_debug(1 << CXGBI_DBG_ISCSI,
2602 "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n",
2603 shost, chba, chba->ndev->name, param, buflen, buf);
2604
2605 switch (param) {
2606 case ISCSI_HOST_PARAM_IPADDRESS:
2607 {
2608 __be32 addr = in_aton(buf);
2609 log_debug(1 << CXGBI_DBG_ISCSI,
2610 "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr);
2611 cxgbi_set_iscsi_ipv4(chba, addr);
2612 return 0;
2613 }
2614 case ISCSI_HOST_PARAM_HWADDRESS:
2615 case ISCSI_HOST_PARAM_NETDEV_NAME:
2616 return 0;
2617 default:
2618 return iscsi_host_set_param(shost, param, buf, buflen);
2619 }
2620}
2621EXPORT_SYMBOL_GPL(cxgbi_set_host_param);
2622
2623int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
2624 char *buf)
2625{
2626 struct cxgbi_hba *chba = iscsi_host_priv(shost);
2627 int len = 0;
2628
2629 if (!chba->ndev) {
2630 shost_printk(KERN_ERR, shost, "Could not get host param. "
2631 "netdev for host not set.\n");
2632 return -ENODEV;
2633 }
2634
2635 log_debug(1 << CXGBI_DBG_ISCSI,
2636 "shost 0x%p, hba 0x%p,%s, param %d.\n",
2637 shost, chba, chba->ndev->name, param);
2638
2639 switch (param) {
2640 case ISCSI_HOST_PARAM_HWADDRESS:
2641 len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6);
2642 break;
2643 case ISCSI_HOST_PARAM_NETDEV_NAME:
2644 len = sprintf(buf, "%s\n", chba->ndev->name);
2645 break;
2646 case ISCSI_HOST_PARAM_IPADDRESS:
2647 {
2648 __be32 addr;
2649
2650 addr = cxgbi_get_iscsi_ipv4(chba);
2651 len = sprintf(buf, "%pI4", &addr);
2652 log_debug(1 << CXGBI_DBG_ISCSI,
2653 "hba %s, ipv4 %pI4.\n", chba->ndev->name, &addr);
2654 break;
2655 }
2656 default:
2657 return iscsi_host_get_param(shost, param, buf);
2658 }
2659
2660 return len;
2661}
2662EXPORT_SYMBOL_GPL(cxgbi_get_host_param);
2663
2664struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
2665 struct sockaddr *dst_addr,
2666 int non_blocking)
2667{
2668 struct iscsi_endpoint *ep;
2669 struct cxgbi_endpoint *cep;
2670 struct cxgbi_hba *hba = NULL;
2671 struct cxgbi_sock *csk;
2672 int err = -EINVAL;
2673
2674 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2675 "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n",
2676 shost, non_blocking, dst_addr);
2677
2678 if (shost) {
2679 hba = iscsi_host_priv(shost);
2680 if (!hba) {
2681 pr_info("shost 0x%p, priv NULL.\n", shost);
2682 goto err_out;
2683 }
2684 }
2685
fc8d0590
AB		 2686 if (dst_addr->sa_family == AF_INET) {
2687 csk = cxgbi_check_route(dst_addr);
e81fbf6c 2688#if IS_ENABLED(CONFIG_IPV6)
fc8d0590
AB		 2689 } else if (dst_addr->sa_family == AF_INET6) {
2690 csk = cxgbi_check_route6(dst_addr);
e81fbf6c 2691#endif
fc8d0590
AB		 2692 } else {
2693 pr_info("address family 0x%x NOT supported.\n",
2694 dst_addr->sa_family);
2695 err = -EAFNOSUPPORT;
2696 return (struct iscsi_endpoint *)ERR_PTR(err);
2697 }
2698
9ba682f0 2699 if (IS_ERR(csk))
2700 return (struct iscsi_endpoint *)csk;
2701 cxgbi_sock_get(csk);
2702
2703 if (!hba)
2704 hba = csk->cdev->hbas[csk->port_id];
2705 else if (hba != csk->cdev->hbas[csk->port_id]) {
2706 pr_info("Could not connect through requested host %u"
2707 "hba 0x%p != 0x%p (%u).\n",
2708 shost->host_no, hba,
2709 csk->cdev->hbas[csk->port_id], csk->port_id);
2710 err = -ENOSPC;
2711 goto release_conn;
2712 }
2713
2714 err = sock_get_port(csk);
2715 if (err)
2716 goto release_conn;
2717
2718 cxgbi_sock_set_state(csk, CTP_CONNECTING);
2719 err = csk->cdev->csk_init_act_open(csk);
2720 if (err)
2721 goto release_conn;
2722
2723 if (cxgbi_sock_is_closing(csk)) {
2724 err = -ENOSPC;
2725 pr_info("csk 0x%p is closing.\n", csk);
2726 goto release_conn;
2727 }
2728
2729 ep = iscsi_create_endpoint(sizeof(*cep));
2730 if (!ep) {
2731 err = -ENOMEM;
2732 pr_info("iscsi alloc ep, OOM.\n");
2733 goto release_conn;
2734 }
2735
2736 cep = ep->dd_data;
2737 cep->csk = csk;
2738 cep->chba = hba;
2739
2740 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2741 "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n",
2742 ep, cep, csk, hba, hba->ndev->name);
2743 return ep;
2744
2745release_conn:
2746 cxgbi_sock_put(csk);
2747 cxgbi_sock_closed(csk);
2748err_out:
2749 return ERR_PTR(err);
2750}
2751EXPORT_SYMBOL_GPL(cxgbi_ep_connect);
2752
2753int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
2754{
2755 struct cxgbi_endpoint *cep = ep->dd_data;
2756 struct cxgbi_sock *csk = cep->csk;
2757
2758 if (!cxgbi_sock_is_established(csk))
2759 return 0;
2760 return 1;
2761}
2762EXPORT_SYMBOL_GPL(cxgbi_ep_poll);
2763
2764void cxgbi_ep_disconnect(struct iscsi_endpoint *ep)
2765{
2766 struct cxgbi_endpoint *cep = ep->dd_data;
2767 struct cxgbi_conn *cconn = cep->cconn;
2768 struct cxgbi_sock *csk = cep->csk;
2769
2770 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2771 "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n",
2772 ep, cep, cconn, csk, csk->state, csk->flags);
2773
2774 if (cconn && cconn->iconn) {
2775 iscsi_suspend_tx(cconn->iconn);
2776 write_lock_bh(&csk->callback_lock);
2777 cep->csk->user_data = NULL;
2778 cconn->cep = NULL;
2779 write_unlock_bh(&csk->callback_lock);
2780 }
2781 iscsi_destroy_endpoint(ep);
2782
2783 if (likely(csk->state >= CTP_ESTABLISHED))
2784 need_active_close(csk);
2785 else
2786 cxgbi_sock_closed(csk);
2787
2788 cxgbi_sock_put(csk);
2789}
2790EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect);
2791
2792int cxgbi_iscsi_init(struct iscsi_transport *itp,
2793 struct scsi_transport_template **stt)
2794{
2795 *stt = iscsi_register_transport(itp);
2796 if (*stt == NULL) {
2797 pr_err("unable to register %s transport 0x%p.\n",
2798 itp->name, itp);
2799 return -ENODEV;
2800 }
2801 log_debug(1 << CXGBI_DBG_ISCSI,
2802 "%s, registered iscsi transport 0x%p.\n",
2803 itp->name, stt);
2804 return 0;
2805}
2806EXPORT_SYMBOL_GPL(cxgbi_iscsi_init);
2807
2808void cxgbi_iscsi_cleanup(struct iscsi_transport *itp,
2809 struct scsi_transport_template **stt)
2810{
2811 if (*stt) {
2812 log_debug(1 << CXGBI_DBG_ISCSI,
2813 "de-register transport 0x%p, %s, stt 0x%p.\n",
2814 itp, itp->name, *stt);
2815 *stt = NULL;
2816 iscsi_unregister_transport(itp);
2817 }
2818}
2819EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup);
2820
587a1f16 2821umode_t cxgbi_attr_is_visible(int param_type, int param)
3128c6c7
MC		 2822{
2823 switch (param_type) {
f27fb2ef
MC		 2824 case ISCSI_HOST_PARAM:
2825 switch (param) {
2826 case ISCSI_HOST_PARAM_NETDEV_NAME:
2827 case ISCSI_HOST_PARAM_HWADDRESS:
2828 case ISCSI_HOST_PARAM_IPADDRESS:
2829 case ISCSI_HOST_PARAM_INITIATOR_NAME:
2830 return S_IRUGO;
2831 default:
2832 return 0;
2833 }
3128c6c7
MC		 2834 case ISCSI_PARAM:
2835 switch (param) {
2836 case ISCSI_PARAM_MAX_RECV_DLENGTH:
2837 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2838 case ISCSI_PARAM_HDRDGST_EN:
2839 case ISCSI_PARAM_DATADGST_EN:
2840 case ISCSI_PARAM_CONN_ADDRESS:
2841 case ISCSI_PARAM_CONN_PORT:
2842 case ISCSI_PARAM_EXP_STATSN:
2843 case ISCSI_PARAM_PERSISTENT_ADDRESS:
2844 case ISCSI_PARAM_PERSISTENT_PORT:
2845 case ISCSI_PARAM_PING_TMO:
2846 case ISCSI_PARAM_RECV_TMO:
1d063c17
MC		 2847 case ISCSI_PARAM_INITIAL_R2T_EN:
2848 case ISCSI_PARAM_MAX_R2T:
2849 case ISCSI_PARAM_IMM_DATA_EN:
2850 case ISCSI_PARAM_FIRST_BURST:
2851 case ISCSI_PARAM_MAX_BURST:
2852 case ISCSI_PARAM_PDU_INORDER_EN:
2853 case ISCSI_PARAM_DATASEQ_INORDER_EN:
2854 case ISCSI_PARAM_ERL:
2855 case ISCSI_PARAM_TARGET_NAME:
2856 case ISCSI_PARAM_TPGT:
2857 case ISCSI_PARAM_USERNAME:
2858 case ISCSI_PARAM_PASSWORD:
2859 case ISCSI_PARAM_USERNAME_IN:
2860 case ISCSI_PARAM_PASSWORD_IN:
2861 case ISCSI_PARAM_FAST_ABORT:
2862 case ISCSI_PARAM_ABORT_TMO:
2863 case ISCSI_PARAM_LU_RESET_TMO:
2864 case ISCSI_PARAM_TGT_RESET_TMO:
2865 case ISCSI_PARAM_IFACE_NAME:
2866 case ISCSI_PARAM_INITIATOR_NAME:
3128c6c7
MC		 2867 return S_IRUGO;
2868 default:
2869 return 0;
2870 }
2871 }
2872
2873 return 0;
2874}
2875EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible);
2876
9ba682f0 2877static int __init libcxgbi_init_module(void)
2878{
2879 sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
2880 sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
2881
2882 pr_info("tag itt 0x%x, %u bits, age 0x%x, %u bits.\n",
2883 ISCSI_ITT_MASK, sw_tag_idx_bits,
2884 ISCSI_AGE_MASK, sw_tag_age_bits);
2885
2886 ddp_setup_host_page_size();
2887 return 0;
2888}
2889
2890static void __exit libcxgbi_exit_module(void)
2891{
2892 cxgbi_device_unregister_all(0xFF);
2893 return;
2894}
2895
2896module_init(libcxgbi_init_module);
2897module_exit(libcxgbi_exit_module);
Ubuntu Artful kernel mirror