]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/scsi/be2iscsi/be_main.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[SCSI] be2iscsi: Update external Branding to Emulex
[mirror_ubuntu-bionic-kernel.git] / drivers / scsi / be2iscsi / be_main.c
CommitLineData
6733b39a 1/**
255fa9a3 2 * Copyright (C) 2005 - 2011 Emulex
6733b39a
JK		 3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
9 *
255fa9a3 10 * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
6733b39a
JK		 11 *
12 * Contact Information:
255fa9a3 13 * linux-drivers@emulex.com
6733b39a 14 *
255fa9a3
JK		 15 * Emulex
16 * 3333 Susan Street
17 * Costa Mesa, CA 92626
6733b39a 18 */
255fa9a3 19
6733b39a
JK		 20#include <linux/reboot.h>
21#include <linux/delay.h>
5a0e3ad6 22#include <linux/slab.h>
6733b39a
JK		 23#include <linux/interrupt.h>
24#include <linux/blkdev.h>
25#include <linux/pci.h>
26#include <linux/string.h>
27#include <linux/kernel.h>
28#include <linux/semaphore.h>
c7acc5b8 29#include <linux/iscsi_boot_sysfs.h>
acf3368f 30#include <linux/module.h>
6733b39a
JK		 31
32#include <scsi/libiscsi.h>
33#include <scsi/scsi_transport_iscsi.h>
34#include <scsi/scsi_transport.h>
35#include <scsi/scsi_cmnd.h>
36#include <scsi/scsi_device.h>
37#include <scsi/scsi_host.h>
38#include <scsi/scsi.h>
39#include "be_main.h"
40#include "be_iscsi.h"
41#include "be_mgmt.h"
42
43static unsigned int be_iopoll_budget = 10;
44static unsigned int be_max_phys_size = 64;
bfead3b2 45static unsigned int enable_msix = 1;
e9b91193
JK		 46static unsigned int gcrashmode = 0;
47static unsigned int num_hba = 0;
6733b39a
JK		 48
49MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
50MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
2f635883 51MODULE_AUTHOR("Emulex Corporation");
6733b39a
JK		 52MODULE_LICENSE("GPL");
53module_param(be_iopoll_budget, int, 0);
54module_param(enable_msix, int, 0);
55module_param(be_max_phys_size, uint, S_IRUGO);
56MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
57 "contiguous memory that can be allocated."
58 "Range is 16 - 128");
59
60static int beiscsi_slave_configure(struct scsi_device *sdev)
61{
62 blk_queue_max_segment_size(sdev->request_queue, 65536);
63 return 0;
64}
65
4183122d
JK		 66static int beiscsi_eh_abort(struct scsi_cmnd *sc)
67{
68 struct iscsi_cls_session *cls_session;
69 struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
70 struct beiscsi_io_task *aborted_io_task;
71 struct iscsi_conn *conn;
72 struct beiscsi_conn *beiscsi_conn;
73 struct beiscsi_hba *phba;
74 struct iscsi_session *session;
75 struct invalidate_command_table *inv_tbl;
3cbb7a74 76 struct be_dma_mem nonemb_cmd;
4183122d
JK		 77 unsigned int cid, tag, num_invalidate;
78
79 cls_session = starget_to_session(scsi_target(sc->device));
80 session = cls_session->dd_data;
81
82 spin_lock_bh(&session->lock);
83 if (!aborted_task || !aborted_task->sc) {
84 /* we raced */
85 spin_unlock_bh(&session->lock);
86 return SUCCESS;
87 }
88
89 aborted_io_task = aborted_task->dd_data;
90 if (!aborted_io_task->scsi_cmnd) {
91 /* raced or invalid command */
92 spin_unlock_bh(&session->lock);
93 return SUCCESS;
94 }
95 spin_unlock_bh(&session->lock);
96 conn = aborted_task->conn;
97 beiscsi_conn = conn->dd_data;
98 phba = beiscsi_conn->phba;
99
100 /* invalidate iocb */
101 cid = beiscsi_conn->beiscsi_conn_cid;
102 inv_tbl = phba->inv_tbl;
103 memset(inv_tbl, 0x0, sizeof(*inv_tbl));
104 inv_tbl->cid = cid;
105 inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
106 num_invalidate = 1;
3cbb7a74
JK		 107 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
108 sizeof(struct invalidate_commands_params_in),
109 &nonemb_cmd.dma);
110 if (nonemb_cmd.va == NULL) {
111 SE_DEBUG(DBG_LVL_1,
112 "Failed to allocate memory for"
113 "mgmt_invalidate_icds\n");
114 return FAILED;
115 }
116 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
117
118 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
119 cid, &nonemb_cmd);
4183122d
JK		 120 if (!tag) {
121 shost_printk(KERN_WARNING, phba->shost,
122 "mgmt_invalidate_icds could not be"
123 " submitted\n");
3cbb7a74
JK		 124 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
125 nonemb_cmd.va, nonemb_cmd.dma);
126
4183122d
JK		 127 return FAILED;
128 } else {
129 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
130 phba->ctrl.mcc_numtag[tag]);
131 free_mcc_tag(&phba->ctrl, tag);
132 }
3cbb7a74
JK		 133 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
134 nonemb_cmd.va, nonemb_cmd.dma);
4183122d
JK		 135 return iscsi_eh_abort(sc);
136}
137
138static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
139{
140 struct iscsi_task *abrt_task;
141 struct beiscsi_io_task *abrt_io_task;
142 struct iscsi_conn *conn;
143 struct beiscsi_conn *beiscsi_conn;
144 struct beiscsi_hba *phba;
145 struct iscsi_session *session;
146 struct iscsi_cls_session *cls_session;
147 struct invalidate_command_table *inv_tbl;
3cbb7a74 148 struct be_dma_mem nonemb_cmd;
4183122d 149 unsigned int cid, tag, i, num_invalidate;
4183122d
JK		 150
151 /* invalidate iocbs */
152 cls_session = starget_to_session(scsi_target(sc->device));
153 session = cls_session->dd_data;
154 spin_lock_bh(&session->lock);
db7f7709
JK		 155 if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
156 spin_unlock_bh(&session->lock);
157 return FAILED;
158 }
4183122d
JK		 159 conn = session->leadconn;
160 beiscsi_conn = conn->dd_data;
161 phba = beiscsi_conn->phba;
162 cid = beiscsi_conn->beiscsi_conn_cid;
163 inv_tbl = phba->inv_tbl;
164 memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
165 num_invalidate = 0;
166 for (i = 0; i < conn->session->cmds_max; i++) {
167 abrt_task = conn->session->cmds[i];
168 abrt_io_task = abrt_task->dd_data;
169 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
170 continue;
171
172 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
173 continue;
174
175 inv_tbl->cid = cid;
176 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
177 num_invalidate++;
178 inv_tbl++;
179 }
180 spin_unlock_bh(&session->lock);
181 inv_tbl = phba->inv_tbl;
182
3cbb7a74
JK		 183 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
184 sizeof(struct invalidate_commands_params_in),
185 &nonemb_cmd.dma);
186 if (nonemb_cmd.va == NULL) {
187 SE_DEBUG(DBG_LVL_1,
188 "Failed to allocate memory for"
189 "mgmt_invalidate_icds\n");
190 return FAILED;
191 }
192 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
193 memset(nonemb_cmd.va, 0, nonemb_cmd.size);
194 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
195 cid, &nonemb_cmd);
4183122d
JK		 196 if (!tag) {
197 shost_printk(KERN_WARNING, phba->shost,
198 "mgmt_invalidate_icds could not be"
199 " submitted\n");
3cbb7a74
JK		 200 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
201 nonemb_cmd.va, nonemb_cmd.dma);
4183122d
JK		 202 return FAILED;
203 } else {
204 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
205 phba->ctrl.mcc_numtag[tag]);
206 free_mcc_tag(&phba->ctrl, tag);
207 }
3cbb7a74
JK		 208 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
209 nonemb_cmd.va, nonemb_cmd.dma);
4183122d 210 return iscsi_eh_device_reset(sc);
4183122d
JK		 211}
212
c7acc5b8
JK		 213static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
214{
215 struct beiscsi_hba *phba = data;
f457a46f
MC		 216 struct mgmt_session_info *boot_sess = &phba->boot_sess;
217 struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
c7acc5b8
JK		 218 char *str = buf;
219 int rc;
220
221 switch (type) {
222 case ISCSI_BOOT_TGT_NAME:
223 rc = sprintf(buf, "%.*s\n",
f457a46f
MC		 224 (int)strlen(boot_sess->target_name),
225 (char *)&boot_sess->target_name);
c7acc5b8
JK		 226 break;
227 case ISCSI_BOOT_TGT_IP_ADDR:
f457a46f 228 if (boot_conn->dest_ipaddr.ip_type == 0x1)
c7acc5b8 229 rc = sprintf(buf, "%pI4\n",
f457a46f 230 (char *)&boot_conn->dest_ipaddr.ip_address);
c7acc5b8
JK		 231 else
232 rc = sprintf(str, "%pI6\n",
f457a46f 233 (char *)&boot_conn->dest_ipaddr.ip_address);
c7acc5b8
JK		 234 break;
235 case ISCSI_BOOT_TGT_PORT:
f457a46f 236 rc = sprintf(str, "%d\n", boot_conn->dest_port);
c7acc5b8
JK		 237 break;
238
239 case ISCSI_BOOT_TGT_CHAP_NAME:
240 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 241 boot_conn->negotiated_login_options.auth_data.chap.
242 target_chap_name_length,
243 (char *)&boot_conn->negotiated_login_options.
244 auth_data.chap.target_chap_name);
c7acc5b8
JK		 245 break;
246 case ISCSI_BOOT_TGT_CHAP_SECRET:
247 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 248 boot_conn->negotiated_login_options.auth_data.chap.
249 target_secret_length,
250 (char *)&boot_conn->negotiated_login_options.
251 auth_data.chap.target_secret);
c7acc5b8
JK		 252 break;
253 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
254 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 255 boot_conn->negotiated_login_options.auth_data.chap.
256 intr_chap_name_length,
257 (char *)&boot_conn->negotiated_login_options.
258 auth_data.chap.intr_chap_name);
c7acc5b8
JK		 259 break;
260 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
f457a46f
MC		 261 rc = sprintf(str, "%.*s\n",
262 boot_conn->negotiated_login_options.auth_data.chap.
263 intr_secret_length,
264 (char *)&boot_conn->negotiated_login_options.
265 auth_data.chap.intr_secret);
c7acc5b8
JK		 266 break;
267 case ISCSI_BOOT_TGT_FLAGS:
f457a46f 268 rc = sprintf(str, "2\n");
c7acc5b8
JK		 269 break;
270 case ISCSI_BOOT_TGT_NIC_ASSOC:
f457a46f 271 rc = sprintf(str, "0\n");
c7acc5b8
JK		 272 break;
273 default:
274 rc = -ENOSYS;
275 break;
276 }
277 return rc;
278}
279
280static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
281{
282 struct beiscsi_hba *phba = data;
283 char *str = buf;
284 int rc;
285
286 switch (type) {
287 case ISCSI_BOOT_INI_INITIATOR_NAME:
288 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
289 break;
290 default:
291 rc = -ENOSYS;
292 break;
293 }
294 return rc;
295}
296
297static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
298{
299 struct beiscsi_hba *phba = data;
300 char *str = buf;
301 int rc;
302
303 switch (type) {
304 case ISCSI_BOOT_ETH_FLAGS:
f457a46f 305 rc = sprintf(str, "2\n");
c7acc5b8
JK		 306 break;
307 case ISCSI_BOOT_ETH_INDEX:
f457a46f 308 rc = sprintf(str, "0\n");
c7acc5b8
JK		 309 break;
310 case ISCSI_BOOT_ETH_MAC:
311 rc = beiscsi_get_macaddr(buf, phba);
312 if (rc < 0) {
313 SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n");
314 return rc;
315 }
316 break;
317 default:
318 rc = -ENOSYS;
319 break;
320 }
321 return rc;
322}
323
324
587a1f16 325static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
c7acc5b8 326{
587a1f16 327 umode_t rc;
c7acc5b8
JK		 328
329 switch (type) {
330 case ISCSI_BOOT_TGT_NAME:
331 case ISCSI_BOOT_TGT_IP_ADDR:
332 case ISCSI_BOOT_TGT_PORT:
333 case ISCSI_BOOT_TGT_CHAP_NAME:
334 case ISCSI_BOOT_TGT_CHAP_SECRET:
335 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
336 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
337 case ISCSI_BOOT_TGT_NIC_ASSOC:
338 case ISCSI_BOOT_TGT_FLAGS:
339 rc = S_IRUGO;
340 break;
341 default:
342 rc = 0;
343 break;
344 }
345 return rc;
346}
347
587a1f16 348static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
c7acc5b8 349{
587a1f16 350 umode_t rc;
c7acc5b8
JK		 351
352 switch (type) {
353 case ISCSI_BOOT_INI_INITIATOR_NAME:
354 rc = S_IRUGO;
355 break;
356 default:
357 rc = 0;
358 break;
359 }
360 return rc;
361}
362
363
587a1f16 364static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
c7acc5b8 365{
587a1f16 366 umode_t rc;
c7acc5b8
JK		 367
368 switch (type) {
369 case ISCSI_BOOT_ETH_FLAGS:
370 case ISCSI_BOOT_ETH_MAC:
371 case ISCSI_BOOT_ETH_INDEX:
372 rc = S_IRUGO;
373 break;
374 default:
375 rc = 0;
376 break;
377 }
378 return rc;
379}
380
bfead3b2
JK		 381/*------------------- PCI Driver operations and data ----------------- */
382static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
383 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
f98c96b0 384 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
bfead3b2
JK		 385 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
386 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
387 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
bfead3b2
JK		 388 { 0 }
389};
390MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
391
6733b39a
JK		 392static struct scsi_host_template beiscsi_sht = {
393 .module = THIS_MODULE,
2f635883 394 .name = "Emulex 10Gbe open-iscsi Initiator Driver",
6733b39a
JK		 395 .proc_name = DRV_NAME,
396 .queuecommand = iscsi_queuecommand,
6733b39a
JK		 397 .change_queue_depth = iscsi_change_queue_depth,
398 .slave_configure = beiscsi_slave_configure,
399 .target_alloc = iscsi_target_alloc,
4183122d
JK		 400 .eh_abort_handler = beiscsi_eh_abort,
401 .eh_device_reset_handler = beiscsi_eh_device_reset,
309ce156 402 .eh_target_reset_handler = iscsi_eh_session_reset,
6733b39a
JK		 403 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
404 .can_queue = BE2_IO_DEPTH,
405 .this_id = -1,
406 .max_sectors = BEISCSI_MAX_SECTORS,
407 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
408 .use_clustering = ENABLE_CLUSTERING,
409};
6733b39a 410
bfead3b2 411static struct scsi_transport_template *beiscsi_scsi_transport;
6733b39a
JK		 412
413static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
414{
415 struct beiscsi_hba *phba;
416 struct Scsi_Host *shost;
417
418 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
419 if (!shost) {
420 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
457ff3b7 421 "iscsi_host_alloc failed\n");
6733b39a
JK		 422 return NULL;
423 }
424 shost->dma_boundary = pcidev->dma_mask;
425 shost->max_id = BE2_MAX_SESSIONS;
426 shost->max_channel = 0;
427 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
428 shost->max_lun = BEISCSI_NUM_MAX_LUN;
429 shost->transportt = beiscsi_scsi_transport;
6733b39a
JK		 430 phba = iscsi_host_priv(shost);
431 memset(phba, 0, sizeof(*phba));
432 phba->shost = shost;
433 phba->pcidev = pci_dev_get(pcidev);
2807afb7 434 pci_set_drvdata(pcidev, phba);
6733b39a
JK		 435
436 if (iscsi_host_add(shost, &phba->pcidev->dev))
437 goto free_devices;
c7acc5b8 438
6733b39a
JK		 439 return phba;
440
441free_devices:
442 pci_dev_put(phba->pcidev);
443 iscsi_host_free(phba->shost);
444 return NULL;
445}
446
447static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
448{
449 if (phba->csr_va) {
450 iounmap(phba->csr_va);
451 phba->csr_va = NULL;
452 }
453 if (phba->db_va) {
454 iounmap(phba->db_va);
455 phba->db_va = NULL;
456 }
457 if (phba->pci_va) {
458 iounmap(phba->pci_va);
459 phba->pci_va = NULL;
460 }
461}
462
463static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
464 struct pci_dev *pcidev)
465{
466 u8 __iomem *addr;
f98c96b0 467 int pcicfg_reg;
6733b39a
JK		 468
469 addr = ioremap_nocache(pci_resource_start(pcidev, 2),
470 pci_resource_len(pcidev, 2));
471 if (addr == NULL)
472 return -ENOMEM;
473 phba->ctrl.csr = addr;
474 phba->csr_va = addr;
475 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
476
477 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
478 if (addr == NULL)
479 goto pci_map_err;
480 phba->ctrl.db = addr;
481 phba->db_va = addr;
482 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
483
f98c96b0
JK		 484 if (phba->generation == BE_GEN2)
485 pcicfg_reg = 1;
486 else
487 pcicfg_reg = 0;
488
489 addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
490 pci_resource_len(pcidev, pcicfg_reg));
491
6733b39a
JK		 492 if (addr == NULL)
493 goto pci_map_err;
494 phba->ctrl.pcicfg = addr;
495 phba->pci_va = addr;
f98c96b0 496 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
6733b39a
JK		 497 return 0;
498
499pci_map_err:
500 beiscsi_unmap_pci_function(phba);
501 return -ENOMEM;
502}
503
504static int beiscsi_enable_pci(struct pci_dev *pcidev)
505{
506 int ret;
507
508 ret = pci_enable_device(pcidev);
509 if (ret) {
510 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
511 "failed. Returning -ENODEV\n");
512 return ret;
513 }
514
bfead3b2 515 pci_set_master(pcidev);
6733b39a
JK		 516 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
517 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
518 if (ret) {
519 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
520 pci_disable_device(pcidev);
521 return ret;
522 }
523 }
524 return 0;
525}
526
527static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
528{
529 struct be_ctrl_info *ctrl = &phba->ctrl;
530 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
531 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
532 int status = 0;
533
534 ctrl->pdev = pdev;
535 status = beiscsi_map_pci_bars(phba, pdev);
536 if (status)
537 return status;
6733b39a
JK		 538 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
539 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
540 mbox_mem_alloc->size,
541 &mbox_mem_alloc->dma);
542 if (!mbox_mem_alloc->va) {
543 beiscsi_unmap_pci_function(phba);
a49e06d5 544 return -ENOMEM;
6733b39a
JK		 545 }
546
547 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
548 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
549 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
550 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
551 spin_lock_init(&ctrl->mbox_lock);
bfead3b2
JK		 552 spin_lock_init(&phba->ctrl.mcc_lock);
553 spin_lock_init(&phba->ctrl.mcc_cq_lock);
554
6733b39a
JK		 555 return status;
556}
557
558static void beiscsi_get_params(struct beiscsi_hba *phba)
559{
7da50879
JK		 560 phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
561 - (phba->fw_config.iscsi_cid_count
562 + BE2_TMFS
563 + BE2_NOPOUT_REQ));
564 phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
ed58ea2a 565 phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
6eab04a8 566 phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
6733b39a
JK		 567 phba->params.num_sge_per_io = BE2_SGE;
568 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
569 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
570 phba->params.eq_timer = 64;
571 phba->params.num_eq_entries =
7da50879
JK		 572 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
573 + BE2_TMFS) / 512) + 1) * 512;
6733b39a
JK		 574 phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
575 ? 1024 : phba->params.num_eq_entries;
457ff3b7 576 SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
7da50879 577 phba->params.num_eq_entries);
6733b39a 578 phba->params.num_cq_entries =
7da50879
JK		 579 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
580 + BE2_TMFS) / 512) + 1) * 512;
6733b39a
JK		 581 phba->params.wrbs_per_cxn = 256;
582}
583
584static void hwi_ring_eq_db(struct beiscsi_hba *phba,
585 unsigned int id, unsigned int clr_interrupt,
586 unsigned int num_processed,
587 unsigned char rearm, unsigned char event)
588{
589 u32 val = 0;
590 val |= id & DB_EQ_RING_ID_MASK;
591 if (rearm)
592 val |= 1 << DB_EQ_REARM_SHIFT;
593 if (clr_interrupt)
594 val |= 1 << DB_EQ_CLR_SHIFT;
595 if (event)
596 val |= 1 << DB_EQ_EVNT_SHIFT;
597 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
598 iowrite32(val, phba->db_va + DB_EQ_OFFSET);
599}
600
bfead3b2
JK		 601/**
602 * be_isr_mcc - The isr routine of the driver.
603 * @irq: Not used
604 * @dev_id: Pointer to host adapter structure
605 */
606static irqreturn_t be_isr_mcc(int irq, void *dev_id)
607{
608 struct beiscsi_hba *phba;
609 struct be_eq_entry *eqe = NULL;
610 struct be_queue_info *eq;
611 struct be_queue_info *mcc;
612 unsigned int num_eq_processed;
613 struct be_eq_obj *pbe_eq;
614 unsigned long flags;
615
616 pbe_eq = dev_id;
617 eq = &pbe_eq->q;
618 phba = pbe_eq->phba;
619 mcc = &phba->ctrl.mcc_obj.cq;
620 eqe = queue_tail_node(eq);
621 if (!eqe)
622 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
623
624 num_eq_processed = 0;
625
626 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
627 & EQE_VALID_MASK) {
628 if (((eqe->dw[offsetof(struct amap_eq_entry,
629 resource_id) / 32] &
630 EQE_RESID_MASK) >> 16) == mcc->id) {
631 spin_lock_irqsave(&phba->isr_lock, flags);
632 phba->todo_mcc_cq = 1;
633 spin_unlock_irqrestore(&phba->isr_lock, flags);
634 }
635 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
636 queue_tail_inc(eq);
637 eqe = queue_tail_node(eq);
638 num_eq_processed++;
639 }
640 if (phba->todo_mcc_cq)
641 queue_work(phba->wq, &phba->work_cqs);
642 if (num_eq_processed)
643 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
644
645 return IRQ_HANDLED;
646}
647
648/**
649 * be_isr_msix - The isr routine of the driver.
650 * @irq: Not used
651 * @dev_id: Pointer to host adapter structure
652 */
653static irqreturn_t be_isr_msix(int irq, void *dev_id)
654{
655 struct beiscsi_hba *phba;
656 struct be_eq_entry *eqe = NULL;
657 struct be_queue_info *eq;
658 struct be_queue_info *cq;
659 unsigned int num_eq_processed;
660 struct be_eq_obj *pbe_eq;
661 unsigned long flags;
662
663 pbe_eq = dev_id;
664 eq = &pbe_eq->q;
665 cq = pbe_eq->cq;
666 eqe = queue_tail_node(eq);
667 if (!eqe)
668 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
669
670 phba = pbe_eq->phba;
671 num_eq_processed = 0;
672 if (blk_iopoll_enabled) {
673 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
674 & EQE_VALID_MASK) {
675 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
676 blk_iopoll_sched(&pbe_eq->iopoll);
677
678 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
679 queue_tail_inc(eq);
680 eqe = queue_tail_node(eq);
681 num_eq_processed++;
682 }
683 if (num_eq_processed)
684 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
685
686 return IRQ_HANDLED;
687 } else {
688 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
689 & EQE_VALID_MASK) {
690 spin_lock_irqsave(&phba->isr_lock, flags);
691 phba->todo_cq = 1;
692 spin_unlock_irqrestore(&phba->isr_lock, flags);
693 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
694 queue_tail_inc(eq);
695 eqe = queue_tail_node(eq);
696 num_eq_processed++;
697 }
698 if (phba->todo_cq)
699 queue_work(phba->wq, &phba->work_cqs);
700
701 if (num_eq_processed)
702 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
703
704 return IRQ_HANDLED;
705 }
706}
707
6733b39a
JK		 708/**
709 * be_isr - The isr routine of the driver.
710 * @irq: Not used
711 * @dev_id: Pointer to host adapter structure
712 */
713static irqreturn_t be_isr(int irq, void *dev_id)
714{
715 struct beiscsi_hba *phba;
716 struct hwi_controller *phwi_ctrlr;
717 struct hwi_context_memory *phwi_context;
718 struct be_eq_entry *eqe = NULL;
719 struct be_queue_info *eq;
720 struct be_queue_info *cq;
bfead3b2 721 struct be_queue_info *mcc;
6733b39a 722 unsigned long flags, index;
bfead3b2 723 unsigned int num_mcceq_processed, num_ioeq_processed;
6733b39a 724 struct be_ctrl_info *ctrl;
bfead3b2 725 struct be_eq_obj *pbe_eq;
6733b39a
JK		 726 int isr;
727
728 phba = dev_id;
6eab04a8 729 ctrl = &phba->ctrl;
bfead3b2
JK		 730 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
731 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
732 if (!isr)
733 return IRQ_NONE;
6733b39a
JK		 734
735 phwi_ctrlr = phba->phwi_ctrlr;
736 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 737 pbe_eq = &phwi_context->be_eq[0];
738
739 eq = &phwi_context->be_eq[0].q;
740 mcc = &phba->ctrl.mcc_obj.cq;
6733b39a
JK		 741 index = 0;
742 eqe = queue_tail_node(eq);
743 if (!eqe)
744 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
745
bfead3b2
JK		 746 num_ioeq_processed = 0;
747 num_mcceq_processed = 0;
6733b39a
JK		 748 if (blk_iopoll_enabled) {
749 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
750 & EQE_VALID_MASK) {
bfead3b2
JK		 751 if (((eqe->dw[offsetof(struct amap_eq_entry,
752 resource_id) / 32] &
753 EQE_RESID_MASK) >> 16) == mcc->id) {
754 spin_lock_irqsave(&phba->isr_lock, flags);
755 phba->todo_mcc_cq = 1;
756 spin_unlock_irqrestore(&phba->isr_lock, flags);
757 num_mcceq_processed++;
758 } else {
759 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
760 blk_iopoll_sched(&pbe_eq->iopoll);
761 num_ioeq_processed++;
762 }
6733b39a
JK		 763 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
764 queue_tail_inc(eq);
765 eqe = queue_tail_node(eq);
6733b39a 766 }
bfead3b2
JK		 767 if (num_ioeq_processed || num_mcceq_processed) {
768 if (phba->todo_mcc_cq)
769 queue_work(phba->wq, &phba->work_cqs);
770
756d29c8 771 if ((num_mcceq_processed) && (!num_ioeq_processed))
bfead3b2
JK		 772 hwi_ring_eq_db(phba, eq->id, 0,
773 (num_ioeq_processed +
774 num_mcceq_processed) , 1, 1);
775 else
776 hwi_ring_eq_db(phba, eq->id, 0,
777 (num_ioeq_processed +
778 num_mcceq_processed), 0, 1);
779
6733b39a
JK		 780 return IRQ_HANDLED;
781 } else
782 return IRQ_NONE;
783 } else {
bfead3b2 784 cq = &phwi_context->be_cq[0];
6733b39a
JK		 785 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
786 & EQE_VALID_MASK) {
787
788 if (((eqe->dw[offsetof(struct amap_eq_entry,
789 resource_id) / 32] &
790 EQE_RESID_MASK) >> 16) != cq->id) {
791 spin_lock_irqsave(&phba->isr_lock, flags);
792 phba->todo_mcc_cq = 1;
793 spin_unlock_irqrestore(&phba->isr_lock, flags);
794 } else {
795 spin_lock_irqsave(&phba->isr_lock, flags);
796 phba->todo_cq = 1;
797 spin_unlock_irqrestore(&phba->isr_lock, flags);
798 }
799 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
800 queue_tail_inc(eq);
801 eqe = queue_tail_node(eq);
bfead3b2 802 num_ioeq_processed++;
6733b39a
JK		 803 }
804 if (phba->todo_cq || phba->todo_mcc_cq)
805 queue_work(phba->wq, &phba->work_cqs);
806
bfead3b2
JK		 807 if (num_ioeq_processed) {
808 hwi_ring_eq_db(phba, eq->id, 0,
809 num_ioeq_processed, 1, 1);
6733b39a
JK		 810 return IRQ_HANDLED;
811 } else
812 return IRQ_NONE;
813 }
814}
815
816static int beiscsi_init_irqs(struct beiscsi_hba *phba)
817{
818 struct pci_dev *pcidev = phba->pcidev;
bfead3b2
JK		 819 struct hwi_controller *phwi_ctrlr;
820 struct hwi_context_memory *phwi_context;
4f5af07e 821 int ret, msix_vec, i, j;
6733b39a 822
bfead3b2
JK		 823 phwi_ctrlr = phba->phwi_ctrlr;
824 phwi_context = phwi_ctrlr->phwi_ctxt;
825
826 if (phba->msix_enabled) {
827 for (i = 0; i < phba->num_cpus; i++) {
8fcfb210
JK		 828 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
829 GFP_KERNEL);
830 if (!phba->msi_name[i]) {
831 ret = -ENOMEM;
832 goto free_msix_irqs;
833 }
834
835 sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
836 phba->shost->host_no, i);
bfead3b2 837 msix_vec = phba->msix_entries[i].vector;
8fcfb210
JK		 838 ret = request_irq(msix_vec, be_isr_msix, 0,
839 phba->msi_name[i],
bfead3b2 840 &phwi_context->be_eq[i]);
4f5af07e
JK		 841 if (ret) {
842 shost_printk(KERN_ERR, phba->shost,
843 "beiscsi_init_irqs-Failed to"
844 "register msix for i = %d\n", i);
8fcfb210 845 kfree(phba->msi_name[i]);
4f5af07e
JK		 846 goto free_msix_irqs;
847 }
bfead3b2 848 }
8fcfb210
JK		 849 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
850 if (!phba->msi_name[i]) {
851 ret = -ENOMEM;
852 goto free_msix_irqs;
853 }
854 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
855 phba->shost->host_no);
bfead3b2 856 msix_vec = phba->msix_entries[i].vector;
8fcfb210 857 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
bfead3b2 858 &phwi_context->be_eq[i]);
4f5af07e
JK		 859 if (ret) {
860 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
861 "Failed to register beiscsi_msix_mcc\n");
8fcfb210 862 kfree(phba->msi_name[i]);
4f5af07e
JK		 863 goto free_msix_irqs;
864 }
865
bfead3b2
JK		 866 } else {
867 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
868 "beiscsi", phba);
869 if (ret) {
870 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
871 "Failed to register irq\\n");
872 return ret;
873 }
6733b39a
JK		 874 }
875 return 0;
4f5af07e 876free_msix_irqs:
8fcfb210
JK		 877 for (j = i - 1; j >= 0; j--) {
878 kfree(phba->msi_name[j]);
879 msix_vec = phba->msix_entries[j].vector;
4f5af07e 880 free_irq(msix_vec, &phwi_context->be_eq[j]);
8fcfb210 881 }
4f5af07e 882 return ret;
6733b39a
JK		 883}
884
885static void hwi_ring_cq_db(struct beiscsi_hba *phba,
886 unsigned int id, unsigned int num_processed,
887 unsigned char rearm, unsigned char event)
888{
889 u32 val = 0;
890 val |= id & DB_CQ_RING_ID_MASK;
891 if (rearm)
892 val |= 1 << DB_CQ_REARM_SHIFT;
893 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
894 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
895}
896
6733b39a
JK		 897static unsigned int
898beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
899 struct beiscsi_hba *phba,
900 unsigned short cid,
901 struct pdu_base *ppdu,
902 unsigned long pdu_len,
903 void *pbuffer, unsigned long buf_len)
904{
905 struct iscsi_conn *conn = beiscsi_conn->conn;
906 struct iscsi_session *session = conn->session;
bfead3b2
JK		 907 struct iscsi_task *task;
908 struct beiscsi_io_task *io_task;
909 struct iscsi_hdr *login_hdr;
6733b39a
JK		 910
911 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
912 PDUBASE_OPCODE_MASK) {
913 case ISCSI_OP_NOOP_IN:
914 pbuffer = NULL;
915 buf_len = 0;
916 break;
917 case ISCSI_OP_ASYNC_EVENT:
918 break;
919 case ISCSI_OP_REJECT:
920 WARN_ON(!pbuffer);
921 WARN_ON(!(buf_len == 48));
922 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
923 break;
924 case ISCSI_OP_LOGIN_RSP:
7bd6e25c 925 case ISCSI_OP_TEXT_RSP:
bfead3b2
JK		 926 task = conn->login_task;
927 io_task = task->dd_data;
928 login_hdr = (struct iscsi_hdr *)ppdu;
929 login_hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 930 break;
931 default:
932 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 933 "Unrecognized opcode 0x%x in async msg\n",
6733b39a
JK		 934 (ppdu->
935 dw[offsetof(struct amap_pdu_base, opcode) / 32]
936 & PDUBASE_OPCODE_MASK));
937 return 1;
938 }
939
940 spin_lock_bh(&session->lock);
941 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
942 spin_unlock_bh(&session->lock);
943 return 0;
944}
945
946static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
947{
948 struct sgl_handle *psgl_handle;
949
950 if (phba->io_sgl_hndl_avbl) {
951 SE_DEBUG(DBG_LVL_8,
457ff3b7 952 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
6733b39a
JK		 953 phba->io_sgl_alloc_index);
954 psgl_handle = phba->io_sgl_hndl_base[phba->
955 io_sgl_alloc_index];
956 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
957 phba->io_sgl_hndl_avbl--;
bfead3b2
JK		 958 if (phba->io_sgl_alloc_index == (phba->params.
959 ios_per_ctrl - 1))
6733b39a
JK		 960 phba->io_sgl_alloc_index = 0;
961 else
962 phba->io_sgl_alloc_index++;
963 } else
964 psgl_handle = NULL;
965 return psgl_handle;
966}
967
968static void
969free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
970{
457ff3b7 971 SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
6733b39a
JK		 972 phba->io_sgl_free_index);
973 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
974 /*
975 * this can happen if clean_task is called on a task that
976 * failed in xmit_task or alloc_pdu.
977 */
978 SE_DEBUG(DBG_LVL_8,
979 "Double Free in IO SGL io_sgl_free_index=%d,"
457ff3b7 980 "value there=%p\n", phba->io_sgl_free_index,
6733b39a
JK		 981 phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
982 return;
983 }
984 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
985 phba->io_sgl_hndl_avbl++;
986 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
987 phba->io_sgl_free_index = 0;
988 else
989 phba->io_sgl_free_index++;
990}
991
992/**
993 * alloc_wrb_handle - To allocate a wrb handle
994 * @phba: The hba pointer
995 * @cid: The cid to use for allocation
6733b39a
JK		 996 *
997 * This happens under session_lock until submission to chip
998 */
d5431488 999struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
6733b39a
JK		 1000{
1001 struct hwi_wrb_context *pwrb_context;
1002 struct hwi_controller *phwi_ctrlr;
d5431488 1003 struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
6733b39a
JK		 1004
1005 phwi_ctrlr = phba->phwi_ctrlr;
1006 pwrb_context = &phwi_ctrlr->wrb_context[cid];
d5431488 1007 if (pwrb_context->wrb_handles_available >= 2) {
bfead3b2
JK		 1008 pwrb_handle = pwrb_context->pwrb_handle_base[
1009 pwrb_context->alloc_index];
1010 pwrb_context->wrb_handles_available--;
bfead3b2
JK		 1011 if (pwrb_context->alloc_index ==
1012 (phba->params.wrbs_per_cxn - 1))
1013 pwrb_context->alloc_index = 0;
1014 else
1015 pwrb_context->alloc_index++;
d5431488
JK		 1016 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1017 pwrb_context->alloc_index];
1018 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
bfead3b2
JK		 1019 } else
1020 pwrb_handle = NULL;
6733b39a
JK		 1021 return pwrb_handle;
1022}
1023
1024/**
1025 * free_wrb_handle - To free the wrb handle back to pool
1026 * @phba: The hba pointer
1027 * @pwrb_context: The context to free from
1028 * @pwrb_handle: The wrb_handle to free
1029 *
1030 * This happens under session_lock until submission to chip
1031 */
1032static void
1033free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1034 struct wrb_handle *pwrb_handle)
1035{
32951dd8 1036 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
bfead3b2
JK		 1037 pwrb_context->wrb_handles_available++;
1038 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1039 pwrb_context->free_index = 0;
1040 else
1041 pwrb_context->free_index++;
1042
6733b39a 1043 SE_DEBUG(DBG_LVL_8,
bfead3b2 1044 "FREE WRB: pwrb_handle=%p free_index=0x%x"
457ff3b7 1045 "wrb_handles_available=%d\n",
6733b39a 1046 pwrb_handle, pwrb_context->free_index,
bfead3b2 1047 pwrb_context->wrb_handles_available);
6733b39a
JK		 1048}
1049
1050static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1051{
1052 struct sgl_handle *psgl_handle;
1053
1054 if (phba->eh_sgl_hndl_avbl) {
1055 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1056 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
457ff3b7 1057 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
6733b39a
JK		 1058 phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
1059 phba->eh_sgl_hndl_avbl--;
1060 if (phba->eh_sgl_alloc_index ==
1061 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1062 1))
1063 phba->eh_sgl_alloc_index = 0;
1064 else
1065 phba->eh_sgl_alloc_index++;
1066 } else
1067 psgl_handle = NULL;
1068 return psgl_handle;
1069}
1070
1071void
1072free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1073{
1074
457ff3b7 1075 SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
bfead3b2 1076 phba->eh_sgl_free_index);
6733b39a
JK		 1077 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1078 /*
1079 * this can happen if clean_task is called on a task that
1080 * failed in xmit_task or alloc_pdu.
1081 */
1082 SE_DEBUG(DBG_LVL_8,
457ff3b7 1083 "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
6733b39a
JK		 1084 phba->eh_sgl_free_index);
1085 return;
1086 }
1087 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1088 phba->eh_sgl_hndl_avbl++;
1089 if (phba->eh_sgl_free_index ==
1090 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1091 phba->eh_sgl_free_index = 0;
1092 else
1093 phba->eh_sgl_free_index++;
1094}
1095
1096static void
1097be_complete_io(struct beiscsi_conn *beiscsi_conn,
1098 struct iscsi_task *task, struct sol_cqe *psol)
1099{
1100 struct beiscsi_io_task *io_task = task->dd_data;
1101 struct be_status_bhs *sts_bhs =
1102 (struct be_status_bhs *)io_task->cmd_bhs;
1103 struct iscsi_conn *conn = beiscsi_conn->conn;
6733b39a
JK		 1104 unsigned char *sense;
1105 u32 resid = 0, exp_cmdsn, max_cmdsn;
1106 u8 rsp, status, flags;
1107
bfead3b2 1108 exp_cmdsn = (psol->
6733b39a
JK		 1109 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1110 & SOL_EXP_CMD_SN_MASK);
bfead3b2 1111 max_cmdsn = ((psol->
6733b39a
JK		 1112 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1113 & SOL_EXP_CMD_SN_MASK) +
1114 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1115 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1116 rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
1117 & SOL_RESP_MASK) >> 16);
1118 status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
1119 & SOL_STS_MASK) >> 8);
1120 flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1121 & SOL_FLAGS_MASK) >> 24) | 0x80;
bd535451
JK		 1122 if (!task->sc) {
1123 if (io_task->scsi_cmnd)
1124 scsi_dma_unmap(io_task->scsi_cmnd);
6733b39a 1125
bd535451
JK		 1126 return;
1127 }
6733b39a
JK		 1128 task->sc->result = (DID_OK << 16) | status;
1129 if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1130 task->sc->result = DID_ERROR << 16;
1131 goto unmap;
1132 }
1133
1134 /* bidi not initially supported */
1135 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1136 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
1137 32] & SOL_RES_CNT_MASK);
1138
1139 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1140 task->sc->result = DID_ERROR << 16;
1141
1142 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1143 scsi_set_resid(task->sc, resid);
1144 if (!status && (scsi_bufflen(task->sc) - resid <
1145 task->sc->underflow))
1146 task->sc->result = DID_ERROR << 16;
1147 }
1148 }
1149
1150 if (status == SAM_STAT_CHECK_CONDITION) {
4053a4be 1151 u16 sense_len;
bfead3b2 1152 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
4053a4be 1153
6733b39a 1154 sense = sts_bhs->sense_info + sizeof(unsigned short);
4053a4be 1155 sense_len = be16_to_cpu(*slen);
6733b39a
JK		 1156 memcpy(task->sc->sense_buffer, sense,
1157 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1158 }
756d29c8 1159
6733b39a
JK		 1160 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
1161 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1162 & SOL_RES_CNT_MASK)
1163 conn->rxdata_octets += (psol->
bfead3b2
JK		 1164 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1165 & SOL_RES_CNT_MASK);
6733b39a
JK		 1166 }
1167unmap:
1168 scsi_dma_unmap(io_task->scsi_cmnd);
1169 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1170}
1171
1172static void
1173be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1174 struct iscsi_task *task, struct sol_cqe *psol)
1175{
1176 struct iscsi_logout_rsp *hdr;
bfead3b2 1177 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1178 struct iscsi_conn *conn = beiscsi_conn->conn;
1179
1180 hdr = (struct iscsi_logout_rsp *)task->hdr;
7bd6e25c 1181 hdr->opcode = ISCSI_OP_LOGOUT_RSP;
6733b39a
JK		 1182 hdr->t2wait = 5;
1183 hdr->t2retain = 0;
1184 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1185 & SOL_FLAGS_MASK) >> 24) | 0x80;
1186 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1187 32] & SOL_RESP_MASK);
1188 hdr->exp_cmdsn = cpu_to_be32(psol->
1189 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1190 & SOL_EXP_CMD_SN_MASK);
1191 hdr->max_cmdsn = be32_to_cpu((psol->
1192 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1193 & SOL_EXP_CMD_SN_MASK) +
1194 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1195 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
7bd6e25c
JK		 1196 hdr->dlength[0] = 0;
1197 hdr->dlength[1] = 0;
1198 hdr->dlength[2] = 0;
6733b39a 1199 hdr->hlength = 0;
bfead3b2 1200 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1201 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1202}
1203
1204static void
1205be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1206 struct iscsi_task *task, struct sol_cqe *psol)
1207{
1208 struct iscsi_tm_rsp *hdr;
1209 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 1210 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1211
1212 hdr = (struct iscsi_tm_rsp *)task->hdr;
7bd6e25c 1213 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
6733b39a
JK		 1214 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1215 & SOL_FLAGS_MASK) >> 24) | 0x80;
1216 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1217 32] & SOL_RESP_MASK);
1218 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
bfead3b2 1219 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
6733b39a
JK		 1220 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1221 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1222 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1223 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
bfead3b2 1224 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1225 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1226}
1227
1228static void
1229hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1230 struct beiscsi_hba *phba, struct sol_cqe *psol)
1231{
1232 struct hwi_wrb_context *pwrb_context;
bfead3b2 1233 struct wrb_handle *pwrb_handle = NULL;
6733b39a 1234 struct hwi_controller *phwi_ctrlr;
bfead3b2
JK		 1235 struct iscsi_task *task;
1236 struct beiscsi_io_task *io_task;
6733b39a
JK		 1237 struct iscsi_conn *conn = beiscsi_conn->conn;
1238 struct iscsi_session *session = conn->session;
1239
1240 phwi_ctrlr = phba->phwi_ctrlr;
32951dd8 1241 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
35e66019 1242 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
7da50879
JK		 1243 SOL_CID_MASK) >> 6) -
1244 phba->fw_config.iscsi_cid_start];
32951dd8 1245 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
35e66019
JK		 1246 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1247 32] & SOL_WRB_INDEX_MASK) >> 16)];
32951dd8 1248 task = pwrb_handle->pio_handle;
35e66019 1249
bfead3b2 1250 io_task = task->dd_data;
1282ab76 1251 spin_lock_bh(&phba->mgmt_sgl_lock);
bfead3b2 1252 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1282ab76 1253 spin_unlock_bh(&phba->mgmt_sgl_lock);
6733b39a
JK		 1254 spin_lock_bh(&session->lock);
1255 free_wrb_handle(phba, pwrb_context, pwrb_handle);
1256 spin_unlock_bh(&session->lock);
1257}
1258
1259static void
1260be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1261 struct iscsi_task *task, struct sol_cqe *psol)
1262{
1263 struct iscsi_nopin *hdr;
1264 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 1265 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1266
1267 hdr = (struct iscsi_nopin *)task->hdr;
1268 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1269 & SOL_FLAGS_MASK) >> 24) | 0x80;
1270 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1271 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1272 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1273 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1274 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1275 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1276 hdr->opcode = ISCSI_OP_NOOP_IN;
bfead3b2 1277 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1278 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1279}
1280
1281static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1282 struct beiscsi_hba *phba, struct sol_cqe *psol)
1283{
1284 struct hwi_wrb_context *pwrb_context;
1285 struct wrb_handle *pwrb_handle;
1286 struct iscsi_wrb *pwrb = NULL;
1287 struct hwi_controller *phwi_ctrlr;
1288 struct iscsi_task *task;
bfead3b2 1289 unsigned int type;
6733b39a
JK		 1290 struct iscsi_conn *conn = beiscsi_conn->conn;
1291 struct iscsi_session *session = conn->session;
1292
1293 phwi_ctrlr = phba->phwi_ctrlr;
32951dd8 1294 pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
35e66019 1295 (struct amap_sol_cqe, cid) / 32]
7da50879
JK		 1296 & SOL_CID_MASK) >> 6) -
1297 phba->fw_config.iscsi_cid_start];
32951dd8 1298 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
35e66019
JK		 1299 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1300 32] & SOL_WRB_INDEX_MASK) >> 16)];
32951dd8
JK		 1301 task = pwrb_handle->pio_handle;
1302 pwrb = pwrb_handle->pwrb;
1303 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1304 WRB_TYPE_MASK) >> 28;
1305
bfead3b2
JK		 1306 spin_lock_bh(&session->lock);
1307 switch (type) {
6733b39a
JK		 1308 case HWH_TYPE_IO:
1309 case HWH_TYPE_IO_RD:
1310 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
dafab8e0 1311 ISCSI_OP_NOOP_OUT)
6733b39a 1312 be_complete_nopin_resp(beiscsi_conn, task, psol);
dafab8e0 1313 else
6733b39a
JK		 1314 be_complete_io(beiscsi_conn, task, psol);
1315 break;
1316
1317 case HWH_TYPE_LOGOUT:
dafab8e0
JK		 1318 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1319 be_complete_logout(beiscsi_conn, task, psol);
1320 else
1321 be_complete_tmf(beiscsi_conn, task, psol);
1322
6733b39a
JK		 1323 break;
1324
1325 case HWH_TYPE_LOGIN:
1326 SE_DEBUG(DBG_LVL_1,
1327 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
457ff3b7 1328 "- Solicited path\n");
6733b39a
JK		 1329 break;
1330
6733b39a
JK		 1331 case HWH_TYPE_NOP:
1332 be_complete_nopin_resp(beiscsi_conn, task, psol);
1333 break;
1334
1335 default:
32951dd8 1336 shost_printk(KERN_WARNING, phba->shost,
35e66019
JK		 1337 "In hwi_complete_cmd, unknown type = %d"
1338 "wrb_index 0x%x CID 0x%x\n", type,
1339 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1340 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1341 ((psol->dw[offsetof(struct amap_sol_cqe,
1342 cid) / 32] & SOL_CID_MASK) >> 6));
6733b39a
JK		 1343 break;
1344 }
35e66019 1345
6733b39a
JK		 1346 spin_unlock_bh(&session->lock);
1347}
1348
1349static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1350 *pasync_ctx, unsigned int is_header,
1351 unsigned int host_write_ptr)
1352{
1353 if (is_header)
1354 return &pasync_ctx->async_entry[host_write_ptr].
1355 header_busy_list;
1356 else
1357 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1358}
1359
1360static struct async_pdu_handle *
1361hwi_get_async_handle(struct beiscsi_hba *phba,
1362 struct beiscsi_conn *beiscsi_conn,
1363 struct hwi_async_pdu_context *pasync_ctx,
1364 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1365{
1366 struct be_bus_address phys_addr;
1367 struct list_head *pbusy_list;
1368 struct async_pdu_handle *pasync_handle = NULL;
6733b39a
JK		 1369 unsigned char is_header = 0;
1370
1371 phys_addr.u.a32.address_lo =
1372 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1373 ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1374 & PDUCQE_DPL_MASK) >> 16);
1375 phys_addr.u.a32.address_hi =
1376 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1377
1378 phys_addr.u.a64.address =
1379 *((unsigned long long *)(&phys_addr.u.a64.address));
1380
1381 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1382 & PDUCQE_CODE_MASK) {
1383 case UNSOL_HDR_NOTIFY:
1384 is_header = 1;
1385
1386 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1387 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1388 index) / 32] & PDUCQE_INDEX_MASK));
6733b39a
JK		 1389 break;
1390 case UNSOL_DATA_NOTIFY:
1391 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1392 dw[offsetof(struct amap_i_t_dpdu_cqe,
1393 index) / 32] & PDUCQE_INDEX_MASK));
6733b39a
JK		 1394 break;
1395 default:
1396 pbusy_list = NULL;
1397 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 1398 "Unexpected code=%d\n",
6733b39a
JK		 1399 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1400 code) / 32] & PDUCQE_CODE_MASK);
1401 return NULL;
1402 }
1403
6733b39a
JK		 1404 WARN_ON(list_empty(pbusy_list));
1405 list_for_each_entry(pasync_handle, pbusy_list, link) {
dc63aac6 1406 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
6733b39a
JK		 1407 break;
1408 }
1409
1410 WARN_ON(!pasync_handle);
1411
7da50879
JK		 1412 pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1413 phba->fw_config.iscsi_cid_start;
6733b39a
JK		 1414 pasync_handle->is_header = is_header;
1415 pasync_handle->buffer_len = ((pdpdu_cqe->
1416 dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1417 & PDUCQE_DPL_MASK) >> 16);
1418
1419 *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1420 index) / 32] & PDUCQE_INDEX_MASK);
1421 return pasync_handle;
1422}
1423
1424static unsigned int
1425hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1426 unsigned int is_header, unsigned int cq_index)
1427{
1428 struct list_head *pbusy_list;
1429 struct async_pdu_handle *pasync_handle;
1430 unsigned int num_entries, writables = 0;
1431 unsigned int *pep_read_ptr, *pwritables;
1432
dc63aac6 1433 num_entries = pasync_ctx->num_entries;
6733b39a
JK		 1434 if (is_header) {
1435 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1436 pwritables = &pasync_ctx->async_header.writables;
6733b39a
JK		 1437 } else {
1438 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1439 pwritables = &pasync_ctx->async_data.writables;
6733b39a
JK		 1440 }
1441
1442 while ((*pep_read_ptr) != cq_index) {
1443 (*pep_read_ptr)++;
1444 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1445
1446 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1447 *pep_read_ptr);
1448 if (writables == 0)
1449 WARN_ON(list_empty(pbusy_list));
1450
1451 if (!list_empty(pbusy_list)) {
1452 pasync_handle = list_entry(pbusy_list->next,
1453 struct async_pdu_handle,
1454 link);
1455 WARN_ON(!pasync_handle);
1456 pasync_handle->consumed = 1;
1457 }
1458
1459 writables++;
1460 }
1461
1462 if (!writables) {
1463 SE_DEBUG(DBG_LVL_1,
1464 "Duplicate notification received - index 0x%x!!\n",
1465 cq_index);
1466 WARN_ON(1);
1467 }
1468
1469 *pwritables = *pwritables + writables;
1470 return 0;
1471}
1472
1473static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1474 unsigned int cri)
1475{
1476 struct hwi_controller *phwi_ctrlr;
1477 struct hwi_async_pdu_context *pasync_ctx;
1478 struct async_pdu_handle *pasync_handle, *tmp_handle;
1479 struct list_head *plist;
1480 unsigned int i = 0;
1481
1482 phwi_ctrlr = phba->phwi_ctrlr;
1483 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1484
1485 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1486
1487 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1488 list_del(&pasync_handle->link);
1489
1490 if (i == 0) {
1491 list_add_tail(&pasync_handle->link,
1492 &pasync_ctx->async_header.free_list);
1493 pasync_ctx->async_header.free_entries++;
1494 i++;
1495 } else {
1496 list_add_tail(&pasync_handle->link,
1497 &pasync_ctx->async_data.free_list);
1498 pasync_ctx->async_data.free_entries++;
1499 i++;
1500 }
1501 }
1502
1503 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1504 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1505 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1506 return 0;
1507}
1508
1509static struct phys_addr *
1510hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1511 unsigned int is_header, unsigned int host_write_ptr)
1512{
1513 struct phys_addr *pasync_sge = NULL;
1514
1515 if (is_header)
1516 pasync_sge = pasync_ctx->async_header.ring_base;
1517 else
1518 pasync_sge = pasync_ctx->async_data.ring_base;
1519
1520 return pasync_sge + host_write_ptr;
1521}
1522
1523static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1524 unsigned int is_header)
1525{
1526 struct hwi_controller *phwi_ctrlr;
1527 struct hwi_async_pdu_context *pasync_ctx;
1528 struct async_pdu_handle *pasync_handle;
1529 struct list_head *pfree_link, *pbusy_list;
1530 struct phys_addr *pasync_sge;
1531 unsigned int ring_id, num_entries;
1532 unsigned int host_write_num;
1533 unsigned int writables;
1534 unsigned int i = 0;
1535 u32 doorbell = 0;
1536
1537 phwi_ctrlr = phba->phwi_ctrlr;
1538 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
dc63aac6 1539 num_entries = pasync_ctx->num_entries;
6733b39a
JK		 1540
1541 if (is_header) {
6733b39a
JK		 1542 writables = min(pasync_ctx->async_header.writables,
1543 pasync_ctx->async_header.free_entries);
1544 pfree_link = pasync_ctx->async_header.free_list.next;
1545 host_write_num = pasync_ctx->async_header.host_write_ptr;
1546 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1547 } else {
6733b39a
JK		 1548 writables = min(pasync_ctx->async_data.writables,
1549 pasync_ctx->async_data.free_entries);
1550 pfree_link = pasync_ctx->async_data.free_list.next;
1551 host_write_num = pasync_ctx->async_data.host_write_ptr;
1552 ring_id = phwi_ctrlr->default_pdu_data.id;
1553 }
1554
1555 writables = (writables / 8) * 8;
1556 if (writables) {
1557 for (i = 0; i < writables; i++) {
1558 pbusy_list =
1559 hwi_get_async_busy_list(pasync_ctx, is_header,
1560 host_write_num);
1561 pasync_handle =
1562 list_entry(pfree_link, struct async_pdu_handle,
1563 link);
1564 WARN_ON(!pasync_handle);
1565 pasync_handle->consumed = 0;
1566
1567 pfree_link = pfree_link->next;
1568
1569 pasync_sge = hwi_get_ring_address(pasync_ctx,
1570 is_header, host_write_num);
1571
1572 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1573 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1574
1575 list_move(&pasync_handle->link, pbusy_list);
1576
1577 host_write_num++;
1578 host_write_num = host_write_num % num_entries;
1579 }
1580
1581 if (is_header) {
1582 pasync_ctx->async_header.host_write_ptr =
1583 host_write_num;
1584 pasync_ctx->async_header.free_entries -= writables;
1585 pasync_ctx->async_header.writables -= writables;
1586 pasync_ctx->async_header.busy_entries += writables;
1587 } else {
1588 pasync_ctx->async_data.host_write_ptr = host_write_num;
1589 pasync_ctx->async_data.free_entries -= writables;
1590 pasync_ctx->async_data.writables -= writables;
1591 pasync_ctx->async_data.busy_entries += writables;
1592 }
1593
1594 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1595 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1596 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1597 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1598 << DB_DEF_PDU_CQPROC_SHIFT;
1599
1600 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1601 }
1602}
1603
1604static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1605 struct beiscsi_conn *beiscsi_conn,
1606 struct i_t_dpdu_cqe *pdpdu_cqe)
1607{
1608 struct hwi_controller *phwi_ctrlr;
1609 struct hwi_async_pdu_context *pasync_ctx;
1610 struct async_pdu_handle *pasync_handle = NULL;
1611 unsigned int cq_index = -1;
1612
1613 phwi_ctrlr = phba->phwi_ctrlr;
1614 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1615
1616 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1617 pdpdu_cqe, &cq_index);
1618 BUG_ON(pasync_handle->is_header != 0);
1619 if (pasync_handle->consumed == 0)
1620 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1621 cq_index);
1622
1623 hwi_free_async_msg(phba, pasync_handle->cri);
1624 hwi_post_async_buffers(phba, pasync_handle->is_header);
1625}
1626
1627static unsigned int
1628hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1629 struct beiscsi_hba *phba,
1630 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1631{
1632 struct list_head *plist;
1633 struct async_pdu_handle *pasync_handle;
1634 void *phdr = NULL;
1635 unsigned int hdr_len = 0, buf_len = 0;
1636 unsigned int status, index = 0, offset = 0;
1637 void *pfirst_buffer = NULL;
1638 unsigned int num_buf = 0;
1639
1640 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1641
1642 list_for_each_entry(pasync_handle, plist, link) {
1643 if (index == 0) {
1644 phdr = pasync_handle->pbuffer;
1645 hdr_len = pasync_handle->buffer_len;
1646 } else {
1647 buf_len = pasync_handle->buffer_len;
1648 if (!num_buf) {
1649 pfirst_buffer = pasync_handle->pbuffer;
1650 num_buf++;
1651 }
1652 memcpy(pfirst_buffer + offset,
1653 pasync_handle->pbuffer, buf_len);
f2ba02b8 1654 offset += buf_len;
6733b39a
JK		 1655 }
1656 index++;
1657 }
1658
1659 status = beiscsi_process_async_pdu(beiscsi_conn, phba,
7da50879
JK		 1660 (beiscsi_conn->beiscsi_conn_cid -
1661 phba->fw_config.iscsi_cid_start),
1662 phdr, hdr_len, pfirst_buffer,
f2ba02b8 1663 offset);
6733b39a
JK		 1664
1665 if (status == 0)
1666 hwi_free_async_msg(phba, cri);
1667 return 0;
1668}
1669
1670static unsigned int
1671hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1672 struct beiscsi_hba *phba,
1673 struct async_pdu_handle *pasync_handle)
1674{
1675 struct hwi_async_pdu_context *pasync_ctx;
1676 struct hwi_controller *phwi_ctrlr;
1677 unsigned int bytes_needed = 0, status = 0;
1678 unsigned short cri = pasync_handle->cri;
1679 struct pdu_base *ppdu;
1680
1681 phwi_ctrlr = phba->phwi_ctrlr;
1682 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1683
1684 list_del(&pasync_handle->link);
1685 if (pasync_handle->is_header) {
1686 pasync_ctx->async_header.busy_entries--;
1687 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1688 hwi_free_async_msg(phba, cri);
1689 BUG();
1690 }
1691
1692 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1693 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1694 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1695 (unsigned short)pasync_handle->buffer_len;
1696 list_add_tail(&pasync_handle->link,
1697 &pasync_ctx->async_entry[cri].wait_queue.list);
1698
1699 ppdu = pasync_handle->pbuffer;
1700 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1701 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1702 0xFFFF0000) | ((be16_to_cpu((ppdu->
1703 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1704 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1705
1706 if (status == 0) {
1707 pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1708 bytes_needed;
1709
1710 if (bytes_needed == 0)
1711 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1712 pasync_ctx, cri);
1713 }
1714 } else {
1715 pasync_ctx->async_data.busy_entries--;
1716 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1717 list_add_tail(&pasync_handle->link,
1718 &pasync_ctx->async_entry[cri].wait_queue.
1719 list);
1720 pasync_ctx->async_entry[cri].wait_queue.
1721 bytes_received +=
1722 (unsigned short)pasync_handle->buffer_len;
1723
1724 if (pasync_ctx->async_entry[cri].wait_queue.
1725 bytes_received >=
1726 pasync_ctx->async_entry[cri].wait_queue.
1727 bytes_needed)
1728 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1729 pasync_ctx, cri);
1730 }
1731 }
1732 return status;
1733}
1734
1735static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1736 struct beiscsi_hba *phba,
1737 struct i_t_dpdu_cqe *pdpdu_cqe)
1738{
1739 struct hwi_controller *phwi_ctrlr;
1740 struct hwi_async_pdu_context *pasync_ctx;
1741 struct async_pdu_handle *pasync_handle = NULL;
1742 unsigned int cq_index = -1;
1743
1744 phwi_ctrlr = phba->phwi_ctrlr;
1745 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1746 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1747 pdpdu_cqe, &cq_index);
1748
1749 if (pasync_handle->consumed == 0)
1750 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1751 cq_index);
1752 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1753 hwi_post_async_buffers(phba, pasync_handle->is_header);
1754}
1755
756d29c8
JK		 1756static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1757{
1758 struct be_queue_info *mcc_cq;
1759 struct be_mcc_compl *mcc_compl;
1760 unsigned int num_processed = 0;
1761
1762 mcc_cq = &phba->ctrl.mcc_obj.cq;
1763 mcc_compl = queue_tail_node(mcc_cq);
1764 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1765 while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1766
1767 if (num_processed >= 32) {
1768 hwi_ring_cq_db(phba, mcc_cq->id,
1769 num_processed, 0, 0);
1770 num_processed = 0;
1771 }
1772 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1773 /* Interpret flags as an async trailer */
1774 if (is_link_state_evt(mcc_compl->flags))
1775 /* Interpret compl as a async link evt */
1776 beiscsi_async_link_state_process(phba,
1777 (struct be_async_event_link_state *) mcc_compl);
1778 else
1779 SE_DEBUG(DBG_LVL_1,
1780 " Unsupported Async Event, flags"
457ff3b7 1781 " = 0x%08x\n", mcc_compl->flags);
756d29c8
JK		 1782 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1783 be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1784 atomic_dec(&phba->ctrl.mcc_obj.q.used);
1785 }
1786
1787 mcc_compl->flags = 0;
1788 queue_tail_inc(mcc_cq);
1789 mcc_compl = queue_tail_node(mcc_cq);
1790 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1791 num_processed++;
1792 }
1793
1794 if (num_processed > 0)
1795 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1796
1797}
bfead3b2
JK		 1798
1799static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
6733b39a 1800{
6733b39a
JK		 1801 struct be_queue_info *cq;
1802 struct sol_cqe *sol;
1803 struct dmsg_cqe *dmsg;
1804 unsigned int num_processed = 0;
1805 unsigned int tot_nump = 0;
1806 struct beiscsi_conn *beiscsi_conn;
c2462288
JK		 1807 struct beiscsi_endpoint *beiscsi_ep;
1808 struct iscsi_endpoint *ep;
bfead3b2 1809 struct beiscsi_hba *phba;
6733b39a 1810
bfead3b2 1811 cq = pbe_eq->cq;
6733b39a 1812 sol = queue_tail_node(cq);
bfead3b2 1813 phba = pbe_eq->phba;
6733b39a
JK		 1814
1815 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1816 CQE_VALID_MASK) {
1817 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1818
32951dd8 1819 ep = phba->ep_array[(u32) ((sol->
c2462288
JK		 1820 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1821 SOL_CID_MASK) >> 6) -
7da50879 1822 phba->fw_config.iscsi_cid_start];
32951dd8 1823
c2462288
JK		 1824 beiscsi_ep = ep->dd_data;
1825 beiscsi_conn = beiscsi_ep->conn;
756d29c8 1826
6733b39a 1827 if (num_processed >= 32) {
bfead3b2 1828 hwi_ring_cq_db(phba, cq->id,
6733b39a
JK		 1829 num_processed, 0, 0);
1830 tot_nump += num_processed;
1831 num_processed = 0;
1832 }
1833
1834 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1835 32] & CQE_CODE_MASK) {
1836 case SOL_CMD_COMPLETE:
1837 hwi_complete_cmd(beiscsi_conn, phba, sol);
1838 break;
1839 case DRIVERMSG_NOTIFY:
457ff3b7 1840 SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n");
6733b39a
JK		 1841 dmsg = (struct dmsg_cqe *)sol;
1842 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1843 break;
1844 case UNSOL_HDR_NOTIFY:
bfead3b2
JK		 1845 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1846 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1847 (struct i_t_dpdu_cqe *)sol);
1848 break;
6733b39a 1849 case UNSOL_DATA_NOTIFY:
bfead3b2 1850 SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
6733b39a
JK		 1851 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1852 (struct i_t_dpdu_cqe *)sol);
1853 break;
1854 case CXN_INVALIDATE_INDEX_NOTIFY:
1855 case CMD_INVALIDATED_NOTIFY:
1856 case CXN_INVALIDATE_NOTIFY:
1857 SE_DEBUG(DBG_LVL_1,
1858 "Ignoring CQ Error notification for cmd/cxn"
1859 "invalidate\n");
1860 break;
1861 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1862 case CMD_KILLED_INVALID_STATSN_RCVD:
1863 case CMD_KILLED_INVALID_R2T_RCVD:
1864 case CMD_CXN_KILLED_LUN_INVALID:
1865 case CMD_CXN_KILLED_ICD_INVALID:
1866 case CMD_CXN_KILLED_ITT_INVALID:
1867 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1868 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
32951dd8 1869 SE_DEBUG(DBG_LVL_1,
6733b39a
JK		 1870 "CQ Error notification for cmd.. "
1871 "code %d cid 0x%x\n",
1872 sol->dw[offsetof(struct amap_sol_cqe, code) /
1873 32] & CQE_CODE_MASK,
1874 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1875 32] & SOL_CID_MASK));
1876 break;
1877 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1878 SE_DEBUG(DBG_LVL_1,
1879 "Digest error on def pdu ring, dropping..\n");
1880 hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1881 (struct i_t_dpdu_cqe *) sol);
1882 break;
1883 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1884 case CXN_KILLED_BURST_LEN_MISMATCH:
1885 case CXN_KILLED_AHS_RCVD:
1886 case CXN_KILLED_HDR_DIGEST_ERR:
1887 case CXN_KILLED_UNKNOWN_HDR:
1888 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1889 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1890 case CXN_KILLED_TIMED_OUT:
1891 case CXN_KILLED_FIN_RCVD:
1892 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1893 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1894 case CXN_KILLED_OVER_RUN_RESIDUAL:
1895 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1896 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
32951dd8 1897 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
6733b39a
JK		 1898 "0x%x...\n",
1899 sol->dw[offsetof(struct amap_sol_cqe, code) /
1900 32] & CQE_CODE_MASK,
7da50879
JK		 1901 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1902 32] & CQE_CID_MASK));
6733b39a
JK		 1903 iscsi_conn_failure(beiscsi_conn->conn,
1904 ISCSI_ERR_CONN_FAILED);
1905 break;
1906 case CXN_KILLED_RST_SENT:
1907 case CXN_KILLED_RST_RCVD:
32951dd8 1908 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
bfead3b2 1909 "received/sent on CID 0x%x...\n",
6733b39a
JK		 1910 sol->dw[offsetof(struct amap_sol_cqe, code) /
1911 32] & CQE_CODE_MASK,
7da50879
JK		 1912 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1913 32] & CQE_CID_MASK));
6733b39a
JK		 1914 iscsi_conn_failure(beiscsi_conn->conn,
1915 ISCSI_ERR_CONN_FAILED);
1916 break;
1917 default:
1918 SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1919 "received on CID 0x%x...\n",
1920 sol->dw[offsetof(struct amap_sol_cqe, code) /
1921 32] & CQE_CODE_MASK,
7da50879
JK		 1922 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1923 32] & CQE_CID_MASK));
6733b39a
JK		 1924 break;
1925 }
1926
1927 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1928 queue_tail_inc(cq);
1929 sol = queue_tail_node(cq);
1930 num_processed++;
1931 }
1932
1933 if (num_processed > 0) {
1934 tot_nump += num_processed;
bfead3b2 1935 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
6733b39a
JK		 1936 }
1937 return tot_nump;
1938}
1939
756d29c8 1940void beiscsi_process_all_cqs(struct work_struct *work)
6733b39a
JK		 1941{
1942 unsigned long flags;
bfead3b2
JK		 1943 struct hwi_controller *phwi_ctrlr;
1944 struct hwi_context_memory *phwi_context;
1945 struct be_eq_obj *pbe_eq;
6733b39a
JK		 1946 struct beiscsi_hba *phba =
1947 container_of(work, struct beiscsi_hba, work_cqs);
1948
bfead3b2
JK		 1949 phwi_ctrlr = phba->phwi_ctrlr;
1950 phwi_context = phwi_ctrlr->phwi_ctxt;
1951 if (phba->msix_enabled)
1952 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1953 else
1954 pbe_eq = &phwi_context->be_eq[0];
1955
6733b39a
JK		 1956 if (phba->todo_mcc_cq) {
1957 spin_lock_irqsave(&phba->isr_lock, flags);
1958 phba->todo_mcc_cq = 0;
1959 spin_unlock_irqrestore(&phba->isr_lock, flags);
756d29c8 1960 beiscsi_process_mcc_isr(phba);
6733b39a
JK		 1961 }
1962
1963 if (phba->todo_cq) {
1964 spin_lock_irqsave(&phba->isr_lock, flags);
1965 phba->todo_cq = 0;
1966 spin_unlock_irqrestore(&phba->isr_lock, flags);
bfead3b2 1967 beiscsi_process_cq(pbe_eq);
6733b39a
JK		 1968 }
1969}
1970
1971static int be_iopoll(struct blk_iopoll *iop, int budget)
1972{
1973 static unsigned int ret;
1974 struct beiscsi_hba *phba;
bfead3b2 1975 struct be_eq_obj *pbe_eq;
6733b39a 1976
bfead3b2
JK		 1977 pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1978 ret = beiscsi_process_cq(pbe_eq);
6733b39a 1979 if (ret < budget) {
bfead3b2 1980 phba = pbe_eq->phba;
6733b39a 1981 blk_iopoll_complete(iop);
bfead3b2
JK		 1982 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1983 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
6733b39a
JK		 1984 }
1985 return ret;
1986}
1987
1988static void
1989hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1990 unsigned int num_sg, struct beiscsi_io_task *io_task)
1991{
1992 struct iscsi_sge *psgl;
58ff4bd0 1993 unsigned int sg_len, index;
6733b39a
JK		 1994 unsigned int sge_len = 0;
1995 unsigned long long addr;
1996 struct scatterlist *l_sg;
1997 unsigned int offset;
1998
1999 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2000 io_task->bhs_pa.u.a32.address_lo);
2001 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2002 io_task->bhs_pa.u.a32.address_hi);
2003
2004 l_sg = sg;
48bd86cf
JK		 2005 for (index = 0; (index < num_sg) && (index < 2); index++,
2006 sg = sg_next(sg)) {
6733b39a
JK		 2007 if (index == 0) {
2008 sg_len = sg_dma_len(sg);
2009 addr = (u64) sg_dma_address(sg);
2010 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
457ff3b7 2011 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2012 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
457ff3b7 2013 ((u32)(addr >> 32)));
6733b39a
JK		 2014 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2015 sg_len);
2016 sge_len = sg_len;
6733b39a 2017 } else {
6733b39a
JK		 2018 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2019 pwrb, sge_len);
2020 sg_len = sg_dma_len(sg);
2021 addr = (u64) sg_dma_address(sg);
2022 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
457ff3b7 2023 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2024 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
457ff3b7 2025 ((u32)(addr >> 32)));
6733b39a
JK		 2026 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2027 sg_len);
2028 }
2029 }
2030 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2031 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2032
2033 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2034
2035 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2036 io_task->bhs_pa.u.a32.address_hi);
2037 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2038 io_task->bhs_pa.u.a32.address_lo);
2039
caf818f1
JK		 2040 if (num_sg == 1) {
2041 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2042 1);
2043 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2044 0);
2045 } else if (num_sg == 2) {
2046 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2047 0);
2048 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2049 1);
2050 } else {
2051 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2052 0);
2053 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2054 0);
2055 }
6733b39a
JK		 2056 sg = l_sg;
2057 psgl++;
2058 psgl++;
2059 offset = 0;
48bd86cf 2060 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
6733b39a
JK		 2061 sg_len = sg_dma_len(sg);
2062 addr = (u64) sg_dma_address(sg);
2063 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2064 (addr & 0xFFFFFFFF));
2065 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2066 (addr >> 32));
2067 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2068 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2069 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2070 offset += sg_len;
2071 }
2072 psgl--;
2073 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2074}
2075
2076static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2077{
2078 struct iscsi_sge *psgl;
2079 unsigned long long addr;
2080 struct beiscsi_io_task *io_task = task->dd_data;
2081 struct beiscsi_conn *beiscsi_conn = io_task->conn;
2082 struct beiscsi_hba *phba = beiscsi_conn->phba;
2083
2084 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2085 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2086 io_task->bhs_pa.u.a32.address_lo);
2087 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2088 io_task->bhs_pa.u.a32.address_hi);
2089
2090 if (task->data) {
2091 if (task->data_count) {
2092 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
2093 addr = (u64) pci_map_single(phba->pcidev,
2094 task->data,
2095 task->data_count, 1);
2096 } else {
2097 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2098 addr = 0;
2099 }
2100 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
457ff3b7 2101 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2102 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
457ff3b7 2103 ((u32)(addr >> 32)));
6733b39a
JK		 2104 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2105 task->data_count);
2106
2107 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2108 } else {
2109 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2110 addr = 0;
2111 }
2112
2113 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2114
2115 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2116
2117 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2118 io_task->bhs_pa.u.a32.address_hi);
2119 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2120 io_task->bhs_pa.u.a32.address_lo);
2121 if (task->data) {
2122 psgl++;
2123 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2124 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2125 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2126 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2127 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2128 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2129
2130 psgl++;
2131 if (task->data) {
2132 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
457ff3b7 2133 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2134 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
457ff3b7 2135 ((u32)(addr >> 32)));
6733b39a
JK		 2136 }
2137 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2138 }
2139 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2140}
2141
2142static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2143{
bfead3b2 2144 unsigned int num_cq_pages, num_async_pdu_buf_pages;
6733b39a
JK		 2145 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2146 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2147
2148 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2149 sizeof(struct sol_cqe));
6733b39a
JK		 2150 num_async_pdu_buf_pages =
2151 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2152 phba->params.defpdu_hdr_sz);
2153 num_async_pdu_buf_sgl_pages =
2154 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2155 sizeof(struct phys_addr));
2156 num_async_pdu_data_pages =
2157 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2158 phba->params.defpdu_data_sz);
2159 num_async_pdu_data_sgl_pages =
2160 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2161 sizeof(struct phys_addr));
2162
2163 phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2164
2165 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2166 BE_ISCSI_PDU_HEADER_SIZE;
2167 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2168 sizeof(struct hwi_context_memory);
2169
6733b39a
JK		 2170
2171 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2172 * (phba->params.wrbs_per_cxn)
2173 * phba->params.cxns_per_ctrl;
2174 wrb_sz_per_cxn = sizeof(struct wrb_handle) *
2175 (phba->params.wrbs_per_cxn);
2176 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2177 phba->params.cxns_per_ctrl);
2178
2179 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2180 phba->params.icds_per_ctrl;
2181 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2182 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2183
2184 phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2185 num_async_pdu_buf_pages * PAGE_SIZE;
2186 phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2187 num_async_pdu_data_pages * PAGE_SIZE;
2188 phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2189 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2190 phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2191 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2192 phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2193 phba->params.asyncpdus_per_ctrl *
2194 sizeof(struct async_pdu_handle);
2195 phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2196 phba->params.asyncpdus_per_ctrl *
2197 sizeof(struct async_pdu_handle);
2198 phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2199 sizeof(struct hwi_async_pdu_context) +
2200 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2201}
2202
2203static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2204{
2205 struct be_mem_descriptor *mem_descr;
2206 dma_addr_t bus_add;
2207 struct mem_array *mem_arr, *mem_arr_orig;
2208 unsigned int i, j, alloc_size, curr_alloc_size;
2209
3ec78271 2210 phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
6733b39a
JK		 2211 if (!phba->phwi_ctrlr)
2212 return -ENOMEM;
2213
2214 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2215 GFP_KERNEL);
2216 if (!phba->init_mem) {
2217 kfree(phba->phwi_ctrlr);
2218 return -ENOMEM;
2219 }
2220
2221 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2222 GFP_KERNEL);
2223 if (!mem_arr_orig) {
2224 kfree(phba->init_mem);
2225 kfree(phba->phwi_ctrlr);
2226 return -ENOMEM;
2227 }
2228
2229 mem_descr = phba->init_mem;
2230 for (i = 0; i < SE_MEM_MAX; i++) {
2231 j = 0;
2232 mem_arr = mem_arr_orig;
2233 alloc_size = phba->mem_req[i];
2234 memset(mem_arr, 0, sizeof(struct mem_array) *
2235 BEISCSI_MAX_FRAGS_INIT);
2236 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2237 do {
2238 mem_arr->virtual_address = pci_alloc_consistent(
2239 phba->pcidev,
2240 curr_alloc_size,
2241 &bus_add);
2242 if (!mem_arr->virtual_address) {
2243 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2244 goto free_mem;
2245 if (curr_alloc_size -
2246 rounddown_pow_of_two(curr_alloc_size))
2247 curr_alloc_size = rounddown_pow_of_two
2248 (curr_alloc_size);
2249 else
2250 curr_alloc_size = curr_alloc_size / 2;
2251 } else {
2252 mem_arr->bus_address.u.
2253 a64.address = (__u64) bus_add;
2254 mem_arr->size = curr_alloc_size;
2255 alloc_size -= curr_alloc_size;
2256 curr_alloc_size = min(be_max_phys_size *
2257 1024, alloc_size);
2258 j++;
2259 mem_arr++;
2260 }
2261 } while (alloc_size);
2262 mem_descr->num_elements = j;
2263 mem_descr->size_in_bytes = phba->mem_req[i];
2264 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2265 GFP_KERNEL);
2266 if (!mem_descr->mem_array)
2267 goto free_mem;
2268
2269 memcpy(mem_descr->mem_array, mem_arr_orig,
2270 sizeof(struct mem_array) * j);
2271 mem_descr++;
2272 }
2273 kfree(mem_arr_orig);
2274 return 0;
2275free_mem:
2276 mem_descr->num_elements = j;
2277 while ((i) || (j)) {
2278 for (j = mem_descr->num_elements; j > 0; j--) {
2279 pci_free_consistent(phba->pcidev,
2280 mem_descr->mem_array[j - 1].size,
2281 mem_descr->mem_array[j - 1].
2282 virtual_address,
457ff3b7
JK		 2283 (unsigned long)mem_descr->
2284 mem_array[j - 1].
6733b39a
JK		 2285 bus_address.u.a64.address);
2286 }
2287 if (i) {
2288 i--;
2289 kfree(mem_descr->mem_array);
2290 mem_descr--;
2291 }
2292 }
2293 kfree(mem_arr_orig);
2294 kfree(phba->init_mem);
2295 kfree(phba->phwi_ctrlr);
2296 return -ENOMEM;
2297}
2298
2299static int beiscsi_get_memory(struct beiscsi_hba *phba)
2300{
2301 beiscsi_find_mem_req(phba);
2302 return beiscsi_alloc_mem(phba);
2303}
2304
2305static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2306{
2307 struct pdu_data_out *pdata_out;
2308 struct pdu_nop_out *pnop_out;
2309 struct be_mem_descriptor *mem_descr;
2310
2311 mem_descr = phba->init_mem;
2312 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2313 pdata_out =
2314 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2315 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2316
2317 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2318 IIOC_SCSI_DATA);
2319
2320 pnop_out =
2321 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2322 virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2323
2324 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2325 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2326 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2327 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2328}
2329
3ec78271 2330static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
6733b39a
JK		 2331{
2332 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
3ec78271 2333 struct wrb_handle *pwrb_handle = NULL;
6733b39a
JK		 2334 struct hwi_controller *phwi_ctrlr;
2335 struct hwi_wrb_context *pwrb_context;
3ec78271
JK		 2336 struct iscsi_wrb *pwrb = NULL;
2337 unsigned int num_cxn_wrbh = 0;
2338 unsigned int num_cxn_wrb = 0, j, idx = 0, index;
6733b39a
JK		 2339
2340 mem_descr_wrbh = phba->init_mem;
2341 mem_descr_wrbh += HWI_MEM_WRBH;
2342
2343 mem_descr_wrb = phba->init_mem;
2344 mem_descr_wrb += HWI_MEM_WRB;
6733b39a
JK		 2345 phwi_ctrlr = phba->phwi_ctrlr;
2346
2347 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2348 pwrb_context = &phwi_ctrlr->wrb_context[index];
6733b39a
JK		 2349 pwrb_context->pwrb_handle_base =
2350 kzalloc(sizeof(struct wrb_handle *) *
2351 phba->params.wrbs_per_cxn, GFP_KERNEL);
3ec78271
JK		 2352 if (!pwrb_context->pwrb_handle_base) {
2353 shost_printk(KERN_ERR, phba->shost,
2354 "Mem Alloc Failed. Failing to load\n");
2355 goto init_wrb_hndl_failed;
2356 }
6733b39a
JK		 2357 pwrb_context->pwrb_handle_basestd =
2358 kzalloc(sizeof(struct wrb_handle *) *
2359 phba->params.wrbs_per_cxn, GFP_KERNEL);
3ec78271
JK		 2360 if (!pwrb_context->pwrb_handle_basestd) {
2361 shost_printk(KERN_ERR, phba->shost,
2362 "Mem Alloc Failed. Failing to load\n");
2363 goto init_wrb_hndl_failed;
2364 }
2365 if (!num_cxn_wrbh) {
2366 pwrb_handle =
2367 mem_descr_wrbh->mem_array[idx].virtual_address;
2368 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2369 ((sizeof(struct wrb_handle)) *
2370 phba->params.wrbs_per_cxn));
2371 idx++;
2372 }
2373 pwrb_context->alloc_index = 0;
2374 pwrb_context->wrb_handles_available = 0;
2375 pwrb_context->free_index = 0;
2376
6733b39a 2377 if (num_cxn_wrbh) {
6733b39a
JK		 2378 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2379 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2380 pwrb_context->pwrb_handle_basestd[j] =
2381 pwrb_handle;
2382 pwrb_context->wrb_handles_available++;
bfead3b2 2383 pwrb_handle->wrb_index = j;
6733b39a
JK		 2384 pwrb_handle++;
2385 }
6733b39a
JK		 2386 num_cxn_wrbh--;
2387 }
2388 }
2389 idx = 0;
ed58ea2a 2390 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
6733b39a 2391 pwrb_context = &phwi_ctrlr->wrb_context[index];
3ec78271 2392 if (!num_cxn_wrb) {
6733b39a 2393 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
7c56533c 2394 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
3ec78271
JK		 2395 ((sizeof(struct iscsi_wrb) *
2396 phba->params.wrbs_per_cxn));
2397 idx++;
2398 }
2399
2400 if (num_cxn_wrb) {
6733b39a
JK		 2401 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2402 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2403 pwrb_handle->pwrb = pwrb;
2404 pwrb++;
2405 }
2406 num_cxn_wrb--;
2407 }
2408 }
3ec78271
JK		 2409 return 0;
2410init_wrb_hndl_failed:
2411 for (j = index; j > 0; j--) {
2412 pwrb_context = &phwi_ctrlr->wrb_context[j];
2413 kfree(pwrb_context->pwrb_handle_base);
2414 kfree(pwrb_context->pwrb_handle_basestd);
2415 }
2416 return -ENOMEM;
6733b39a
JK		 2417}
2418
2419static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2420{
2421 struct hwi_controller *phwi_ctrlr;
2422 struct hba_parameters *p = &phba->params;
2423 struct hwi_async_pdu_context *pasync_ctx;
2424 struct async_pdu_handle *pasync_header_h, *pasync_data_h;
dc63aac6 2425 unsigned int index, idx, num_per_mem, num_async_data;
6733b39a
JK		 2426 struct be_mem_descriptor *mem_descr;
2427
2428 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2429 mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2430
2431 phwi_ctrlr = phba->phwi_ctrlr;
2432 phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2433 mem_descr->mem_array[0].virtual_address;
2434 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2435 memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2436
dc63aac6
JK		 2437 pasync_ctx->num_entries = p->asyncpdus_per_ctrl;
2438 pasync_ctx->buffer_size = p->defpdu_hdr_sz;
6733b39a
JK		 2439
2440 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2441 mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2442 if (mem_descr->mem_array[0].virtual_address) {
2443 SE_DEBUG(DBG_LVL_8,
2444 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
457ff3b7 2445 "va=%p\n", mem_descr->mem_array[0].virtual_address);
6733b39a
JK		 2446 } else
2447 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2448 "No Virtual address\n");
6733b39a
JK		 2449
2450 pasync_ctx->async_header.va_base =
2451 mem_descr->mem_array[0].virtual_address;
2452
2453 pasync_ctx->async_header.pa_base.u.a64.address =
2454 mem_descr->mem_array[0].bus_address.u.a64.address;
2455
2456 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2457 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2458 if (mem_descr->mem_array[0].virtual_address) {
2459 SE_DEBUG(DBG_LVL_8,
2460 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
457ff3b7 2461 "va=%p\n", mem_descr->mem_array[0].virtual_address);
6733b39a
JK		 2462 } else
2463 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2464 "No Virtual address\n");
6733b39a
JK		 2465 pasync_ctx->async_header.ring_base =
2466 mem_descr->mem_array[0].virtual_address;
2467
2468 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2469 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2470 if (mem_descr->mem_array[0].virtual_address) {
2471 SE_DEBUG(DBG_LVL_8,
2472 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
457ff3b7 2473 "va=%p\n", mem_descr->mem_array[0].virtual_address);
6733b39a
JK		 2474 } else
2475 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2476 "No Virtual address\n");
6733b39a
JK		 2477
2478 pasync_ctx->async_header.handle_base =
2479 mem_descr->mem_array[0].virtual_address;
2480 pasync_ctx->async_header.writables = 0;
2481 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2482
6733b39a
JK		 2483
2484 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2485 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2486 if (mem_descr->mem_array[0].virtual_address) {
2487 SE_DEBUG(DBG_LVL_8,
2488 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
457ff3b7 2489 "va=%p\n", mem_descr->mem_array[0].virtual_address);
6733b39a
JK		 2490 } else
2491 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2492 "No Virtual address\n");
6733b39a
JK		 2493
2494 pasync_ctx->async_data.ring_base =
2495 mem_descr->mem_array[0].virtual_address;
2496
2497 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2498 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2499 if (!mem_descr->mem_array[0].virtual_address)
2500 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2501 "No Virtual address\n");
6733b39a
JK		 2502
2503 pasync_ctx->async_data.handle_base =
2504 mem_descr->mem_array[0].virtual_address;
2505 pasync_ctx->async_data.writables = 0;
2506 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2507
2508 pasync_header_h =
2509 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2510 pasync_data_h =
2511 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2512
dc63aac6
JK		 2513 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2514 mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2515 if (mem_descr->mem_array[0].virtual_address) {
2516 SE_DEBUG(DBG_LVL_8,
2517 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2518 "va=%p\n", mem_descr->mem_array[0].virtual_address);
2519 } else
2520 shost_printk(KERN_WARNING, phba->shost,
2521 "No Virtual address\n");
2522 idx = 0;
2523 pasync_ctx->async_data.va_base =
2524 mem_descr->mem_array[idx].virtual_address;
2525 pasync_ctx->async_data.pa_base.u.a64.address =
2526 mem_descr->mem_array[idx].bus_address.u.a64.address;
2527
2528 num_async_data = ((mem_descr->mem_array[idx].size) /
2529 phba->params.defpdu_data_sz);
2530 num_per_mem = 0;
2531
6733b39a
JK		 2532 for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2533 pasync_header_h->cri = -1;
2534 pasync_header_h->index = (char)index;
2535 INIT_LIST_HEAD(&pasync_header_h->link);
2536 pasync_header_h->pbuffer =
2537 (void *)((unsigned long)
2538 (pasync_ctx->async_header.va_base) +
2539 (p->defpdu_hdr_sz * index));
2540
2541 pasync_header_h->pa.u.a64.address =
2542 pasync_ctx->async_header.pa_base.u.a64.address +
2543 (p->defpdu_hdr_sz * index);
2544
2545 list_add_tail(&pasync_header_h->link,
2546 &pasync_ctx->async_header.free_list);
2547 pasync_header_h++;
2548 pasync_ctx->async_header.free_entries++;
2549 pasync_ctx->async_header.writables++;
2550
2551 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2552 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2553 header_busy_list);
2554 pasync_data_h->cri = -1;
2555 pasync_data_h->index = (char)index;
2556 INIT_LIST_HEAD(&pasync_data_h->link);
dc63aac6
JK		 2557
2558 if (!num_async_data) {
2559 num_per_mem = 0;
2560 idx++;
2561 pasync_ctx->async_data.va_base =
2562 mem_descr->mem_array[idx].virtual_address;
2563 pasync_ctx->async_data.pa_base.u.a64.address =
2564 mem_descr->mem_array[idx].
2565 bus_address.u.a64.address;
2566
2567 num_async_data = ((mem_descr->mem_array[idx].size) /
2568 phba->params.defpdu_data_sz);
2569 }
6733b39a
JK		 2570 pasync_data_h->pbuffer =
2571 (void *)((unsigned long)
2572 (pasync_ctx->async_data.va_base) +
dc63aac6 2573 (p->defpdu_data_sz * num_per_mem));
6733b39a
JK		 2574
2575 pasync_data_h->pa.u.a64.address =
2576 pasync_ctx->async_data.pa_base.u.a64.address +
dc63aac6
JK		 2577 (p->defpdu_data_sz * num_per_mem);
2578 num_per_mem++;
2579 num_async_data--;
6733b39a
JK		 2580
2581 list_add_tail(&pasync_data_h->link,
2582 &pasync_ctx->async_data.free_list);
2583 pasync_data_h++;
2584 pasync_ctx->async_data.free_entries++;
2585 pasync_ctx->async_data.writables++;
2586
2587 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2588 }
2589
2590 pasync_ctx->async_header.host_write_ptr = 0;
2591 pasync_ctx->async_header.ep_read_ptr = -1;
2592 pasync_ctx->async_data.host_write_ptr = 0;
2593 pasync_ctx->async_data.ep_read_ptr = -1;
2594}
2595
2596static int
2597be_sgl_create_contiguous(void *virtual_address,
2598 u64 physical_address, u32 length,
2599 struct be_dma_mem *sgl)
2600{
2601 WARN_ON(!virtual_address);
2602 WARN_ON(!physical_address);
2603 WARN_ON(!length > 0);
2604 WARN_ON(!sgl);
2605
2606 sgl->va = virtual_address;
457ff3b7 2607 sgl->dma = (unsigned long)physical_address;
6733b39a
JK		 2608 sgl->size = length;
2609
2610 return 0;
2611}
2612
2613static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2614{
2615 memset(sgl, 0, sizeof(*sgl));
2616}
2617
2618static void
2619hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2620 struct mem_array *pmem, struct be_dma_mem *sgl)
2621{
2622 if (sgl->va)
2623 be_sgl_destroy_contiguous(sgl);
2624
2625 be_sgl_create_contiguous(pmem->virtual_address,
2626 pmem->bus_address.u.a64.address,
2627 pmem->size, sgl);
2628}
2629
2630static void
2631hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2632 struct mem_array *pmem, struct be_dma_mem *sgl)
2633{
2634 if (sgl->va)
2635 be_sgl_destroy_contiguous(sgl);
2636
2637 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2638 pmem->bus_address.u.a64.address,
2639 pmem->size, sgl);
2640}
2641
2642static int be_fill_queue(struct be_queue_info *q,
2643 u16 len, u16 entry_size, void *vaddress)
2644{
2645 struct be_dma_mem *mem = &q->dma_mem;
2646
2647 memset(q, 0, sizeof(*q));
2648 q->len = len;
2649 q->entry_size = entry_size;
2650 mem->size = len * entry_size;
2651 mem->va = vaddress;
2652 if (!mem->va)
2653 return -ENOMEM;
2654 memset(mem->va, 0, mem->size);
2655 return 0;
2656}
2657
bfead3b2 2658static int beiscsi_create_eqs(struct beiscsi_hba *phba,
6733b39a
JK		 2659 struct hwi_context_memory *phwi_context)
2660{
bfead3b2
JK		 2661 unsigned int i, num_eq_pages;
2662 int ret, eq_for_mcc;
6733b39a
JK		 2663 struct be_queue_info *eq;
2664 struct be_dma_mem *mem;
6733b39a 2665 void *eq_vaddress;
bfead3b2 2666 dma_addr_t paddr;
6733b39a 2667
bfead3b2
JK		 2668 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2669 sizeof(struct be_eq_entry));
6733b39a 2670
bfead3b2
JK		 2671 if (phba->msix_enabled)
2672 eq_for_mcc = 1;
2673 else
2674 eq_for_mcc = 0;
2675 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2676 eq = &phwi_context->be_eq[i].q;
2677 mem = &eq->dma_mem;
2678 phwi_context->be_eq[i].phba = phba;
2679 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2680 num_eq_pages * PAGE_SIZE,
2681 &paddr);
2682 if (!eq_vaddress)
2683 goto create_eq_error;
2684
2685 mem->va = eq_vaddress;
2686 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2687 sizeof(struct be_eq_entry), eq_vaddress);
2688 if (ret) {
2689 shost_printk(KERN_ERR, phba->shost,
457ff3b7 2690 "be_fill_queue Failed for EQ\n");
bfead3b2
JK		 2691 goto create_eq_error;
2692 }
6733b39a 2693
bfead3b2
JK		 2694 mem->dma = paddr;
2695 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2696 phwi_context->cur_eqd);
2697 if (ret) {
2698 shost_printk(KERN_ERR, phba->shost,
2699 "beiscsi_cmd_eq_create"
457ff3b7 2700 "Failedfor EQ\n");
bfead3b2
JK		 2701 goto create_eq_error;
2702 }
2703 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
6733b39a 2704 }
6733b39a 2705 return 0;
bfead3b2
JK		 2706create_eq_error:
2707 for (i = 0; i < (phba->num_cpus + 1); i++) {
2708 eq = &phwi_context->be_eq[i].q;
2709 mem = &eq->dma_mem;
2710 if (mem->va)
2711 pci_free_consistent(phba->pcidev, num_eq_pages
2712 * PAGE_SIZE,
2713 mem->va, mem->dma);
2714 }
2715 return ret;
6733b39a
JK		 2716}
2717
bfead3b2 2718static int beiscsi_create_cqs(struct beiscsi_hba *phba,
6733b39a
JK		 2719 struct hwi_context_memory *phwi_context)
2720{
bfead3b2 2721 unsigned int i, num_cq_pages;
6733b39a
JK		 2722 int ret;
2723 struct be_queue_info *cq, *eq;
2724 struct be_dma_mem *mem;
bfead3b2 2725 struct be_eq_obj *pbe_eq;
6733b39a 2726 void *cq_vaddress;
bfead3b2 2727 dma_addr_t paddr;
6733b39a 2728
bfead3b2
JK		 2729 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2730 sizeof(struct sol_cqe));
6733b39a 2731
bfead3b2
JK		 2732 for (i = 0; i < phba->num_cpus; i++) {
2733 cq = &phwi_context->be_cq[i];
2734 eq = &phwi_context->be_eq[i].q;
2735 pbe_eq = &phwi_context->be_eq[i];
2736 pbe_eq->cq = cq;
2737 pbe_eq->phba = phba;
2738 mem = &cq->dma_mem;
2739 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2740 num_cq_pages * PAGE_SIZE,
2741 &paddr);
2742 if (!cq_vaddress)
2743 goto create_cq_error;
7da50879 2744 ret = be_fill_queue(cq, phba->params.num_cq_entries,
bfead3b2
JK		 2745 sizeof(struct sol_cqe), cq_vaddress);
2746 if (ret) {
2747 shost_printk(KERN_ERR, phba->shost,
457ff3b7 2748 "be_fill_queue Failed for ISCSI CQ\n");
bfead3b2
JK		 2749 goto create_cq_error;
2750 }
2751
2752 mem->dma = paddr;
2753 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2754 false, 0);
2755 if (ret) {
2756 shost_printk(KERN_ERR, phba->shost,
2757 "beiscsi_cmd_eq_create"
457ff3b7 2758 "Failed for ISCSI CQ\n");
bfead3b2
JK		 2759 goto create_cq_error;
2760 }
2761 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2762 cq->id, eq->id);
2763 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
6733b39a 2764 }
6733b39a 2765 return 0;
bfead3b2
JK		 2766
2767create_cq_error:
2768 for (i = 0; i < phba->num_cpus; i++) {
2769 cq = &phwi_context->be_cq[i];
2770 mem = &cq->dma_mem;
2771 if (mem->va)
2772 pci_free_consistent(phba->pcidev, num_cq_pages
2773 * PAGE_SIZE,
2774 mem->va, mem->dma);
2775 }
2776 return ret;
2777
6733b39a
JK		 2778}
2779
2780static int
2781beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2782 struct hwi_context_memory *phwi_context,
2783 struct hwi_controller *phwi_ctrlr,
2784 unsigned int def_pdu_ring_sz)
2785{
2786 unsigned int idx;
2787 int ret;
2788 struct be_queue_info *dq, *cq;
2789 struct be_dma_mem *mem;
2790 struct be_mem_descriptor *mem_descr;
2791 void *dq_vaddress;
2792
2793 idx = 0;
2794 dq = &phwi_context->be_def_hdrq;
bfead3b2 2795 cq = &phwi_context->be_cq[0];
6733b39a
JK		 2796 mem = &dq->dma_mem;
2797 mem_descr = phba->init_mem;
2798 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2799 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2800 ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2801 sizeof(struct phys_addr),
2802 sizeof(struct phys_addr), dq_vaddress);
2803 if (ret) {
2804 shost_printk(KERN_ERR, phba->shost,
2805 "be_fill_queue Failed for DEF PDU HDR\n");
2806 return ret;
2807 }
457ff3b7
JK		 2808 mem->dma = (unsigned long)mem_descr->mem_array[idx].
2809 bus_address.u.a64.address;
6733b39a
JK		 2810 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2811 def_pdu_ring_sz,
2812 phba->params.defpdu_hdr_sz);
2813 if (ret) {
2814 shost_printk(KERN_ERR, phba->shost,
2815 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2816 return ret;
2817 }
2818 phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2819 SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2820 phwi_context->be_def_hdrq.id);
2821 hwi_post_async_buffers(phba, 1);
2822 return 0;
2823}
2824
2825static int
2826beiscsi_create_def_data(struct beiscsi_hba *phba,
2827 struct hwi_context_memory *phwi_context,
2828 struct hwi_controller *phwi_ctrlr,
2829 unsigned int def_pdu_ring_sz)
2830{
2831 unsigned int idx;
2832 int ret;
2833 struct be_queue_info *dataq, *cq;
2834 struct be_dma_mem *mem;
2835 struct be_mem_descriptor *mem_descr;
2836 void *dq_vaddress;
2837
2838 idx = 0;
2839 dataq = &phwi_context->be_def_dataq;
bfead3b2 2840 cq = &phwi_context->be_cq[0];
6733b39a
JK		 2841 mem = &dataq->dma_mem;
2842 mem_descr = phba->init_mem;
2843 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2844 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2845 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2846 sizeof(struct phys_addr),
2847 sizeof(struct phys_addr), dq_vaddress);
2848 if (ret) {
2849 shost_printk(KERN_ERR, phba->shost,
2850 "be_fill_queue Failed for DEF PDU DATA\n");
2851 return ret;
2852 }
457ff3b7
JK		 2853 mem->dma = (unsigned long)mem_descr->mem_array[idx].
2854 bus_address.u.a64.address;
6733b39a
JK		 2855 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2856 def_pdu_ring_sz,
2857 phba->params.defpdu_data_sz);
2858 if (ret) {
2859 shost_printk(KERN_ERR, phba->shost,
2860 "be_cmd_create_default_pdu_queue Failed"
2861 " for DEF PDU DATA\n");
2862 return ret;
2863 }
2864 phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2865 SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2866 phwi_context->be_def_dataq.id);
2867 hwi_post_async_buffers(phba, 0);
457ff3b7 2868 SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n");
6733b39a
JK		 2869 return 0;
2870}
2871
2872static int
2873beiscsi_post_pages(struct beiscsi_hba *phba)
2874{
2875 struct be_mem_descriptor *mem_descr;
2876 struct mem_array *pm_arr;
2877 unsigned int page_offset, i;
2878 struct be_dma_mem sgl;
2879 int status;
2880
2881 mem_descr = phba->init_mem;
2882 mem_descr += HWI_MEM_SGE;
2883 pm_arr = mem_descr->mem_array;
2884
2885 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2886 phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2887 for (i = 0; i < mem_descr->num_elements; i++) {
2888 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2889 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2890 page_offset,
2891 (pm_arr->size / PAGE_SIZE));
2892 page_offset += pm_arr->size / PAGE_SIZE;
2893 if (status != 0) {
2894 shost_printk(KERN_ERR, phba->shost,
2895 "post sgl failed.\n");
2896 return status;
2897 }
2898 pm_arr++;
2899 }
457ff3b7 2900 SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n");
6733b39a
JK		 2901 return 0;
2902}
2903
bfead3b2
JK		 2904static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2905{
2906 struct be_dma_mem *mem = &q->dma_mem;
c8b25598 2907 if (mem->va) {
bfead3b2
JK		 2908 pci_free_consistent(phba->pcidev, mem->size,
2909 mem->va, mem->dma);
c8b25598
JK		 2910 mem->va = NULL;
2911 }
bfead3b2
JK		 2912}
2913
2914static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2915 u16 len, u16 entry_size)
2916{
2917 struct be_dma_mem *mem = &q->dma_mem;
2918
2919 memset(q, 0, sizeof(*q));
2920 q->len = len;
2921 q->entry_size = entry_size;
2922 mem->size = len * entry_size;
2923 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2924 if (!mem->va)
d3ad2bb3 2925 return -ENOMEM;
bfead3b2
JK		 2926 memset(mem->va, 0, mem->size);
2927 return 0;
2928}
2929
6733b39a
JK		 2930static int
2931beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2932 struct hwi_context_memory *phwi_context,
2933 struct hwi_controller *phwi_ctrlr)
2934{
2935 unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2936 u64 pa_addr_lo;
2937 unsigned int idx, num, i;
2938 struct mem_array *pwrb_arr;
2939 void *wrb_vaddr;
2940 struct be_dma_mem sgl;
2941 struct be_mem_descriptor *mem_descr;
2942 int status;
2943
2944 idx = 0;
2945 mem_descr = phba->init_mem;
2946 mem_descr += HWI_MEM_WRB;
2947 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2948 GFP_KERNEL);
2949 if (!pwrb_arr) {
2950 shost_printk(KERN_ERR, phba->shost,
2951 "Memory alloc failed in create wrb ring.\n");
2952 return -ENOMEM;
2953 }
2954 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2955 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2956 num_wrb_rings = mem_descr->mem_array[idx].size /
2957 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2958
2959 for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2960 if (num_wrb_rings) {
2961 pwrb_arr[num].virtual_address = wrb_vaddr;
2962 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2963 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2964 sizeof(struct iscsi_wrb);
2965 wrb_vaddr += pwrb_arr[num].size;
2966 pa_addr_lo += pwrb_arr[num].size;
2967 num_wrb_rings--;
2968 } else {
2969 idx++;
2970 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2971 pa_addr_lo = mem_descr->mem_array[idx].\
2972 bus_address.u.a64.address;
2973 num_wrb_rings = mem_descr->mem_array[idx].size /
2974 (phba->params.wrbs_per_cxn *
2975 sizeof(struct iscsi_wrb));
2976 pwrb_arr[num].virtual_address = wrb_vaddr;
2977 pwrb_arr[num].bus_address.u.a64.address\
2978 = pa_addr_lo;
2979 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2980 sizeof(struct iscsi_wrb);
2981 wrb_vaddr += pwrb_arr[num].size;
2982 pa_addr_lo += pwrb_arr[num].size;
2983 num_wrb_rings--;
2984 }
2985 }
2986 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2987 wrb_mem_index = 0;
2988 offset = 0;
2989 size = 0;
2990
2991 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2992 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2993 &phwi_context->be_wrbq[i]);
2994 if (status != 0) {
2995 shost_printk(KERN_ERR, phba->shost,
2996 "wrbq create failed.");
1462b8ff 2997 kfree(pwrb_arr);
6733b39a
JK		 2998 return status;
2999 }
7da50879
JK		 3000 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
3001 id;
6733b39a
JK		 3002 }
3003 kfree(pwrb_arr);
3004 return 0;
3005}
3006
3007static void free_wrb_handles(struct beiscsi_hba *phba)
3008{
3009 unsigned int index;
3010 struct hwi_controller *phwi_ctrlr;
3011 struct hwi_wrb_context *pwrb_context;
3012
3013 phwi_ctrlr = phba->phwi_ctrlr;
3014 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3015 pwrb_context = &phwi_ctrlr->wrb_context[index];
3016 kfree(pwrb_context->pwrb_handle_base);
3017 kfree(pwrb_context->pwrb_handle_basestd);
3018 }
3019}
3020
bfead3b2
JK		 3021static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3022{
3023 struct be_queue_info *q;
3024 struct be_ctrl_info *ctrl = &phba->ctrl;
3025
3026 q = &phba->ctrl.mcc_obj.q;
3027 if (q->created)
3028 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3029 be_queue_free(phba, q);
3030
3031 q = &phba->ctrl.mcc_obj.cq;
3032 if (q->created)
3033 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3034 be_queue_free(phba, q);
3035}
3036
6733b39a
JK		 3037static void hwi_cleanup(struct beiscsi_hba *phba)
3038{
3039 struct be_queue_info *q;
3040 struct be_ctrl_info *ctrl = &phba->ctrl;
3041 struct hwi_controller *phwi_ctrlr;
3042 struct hwi_context_memory *phwi_context;
bfead3b2 3043 int i, eq_num;
6733b39a
JK		 3044
3045 phwi_ctrlr = phba->phwi_ctrlr;
3046 phwi_context = phwi_ctrlr->phwi_ctxt;
3047 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3048 q = &phwi_context->be_wrbq[i];
3049 if (q->created)
3050 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3051 }
6733b39a
JK		 3052 free_wrb_handles(phba);
3053
3054 q = &phwi_context->be_def_hdrq;
3055 if (q->created)
3056 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3057
3058 q = &phwi_context->be_def_dataq;
3059 if (q->created)
3060 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3061
3062 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3063
bfead3b2
JK		 3064 for (i = 0; i < (phba->num_cpus); i++) {
3065 q = &phwi_context->be_cq[i];
3066 if (q->created)
3067 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3068 }
3069 if (phba->msix_enabled)
3070 eq_num = 1;
3071 else
3072 eq_num = 0;
3073 for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3074 q = &phwi_context->be_eq[i].q;
3075 if (q->created)
3076 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3077 }
3078 be_mcc_queues_destroy(phba);
3079}
6733b39a 3080
bfead3b2
JK		 3081static int be_mcc_queues_create(struct beiscsi_hba *phba,
3082 struct hwi_context_memory *phwi_context)
3083{
3084 struct be_queue_info *q, *cq;
3085 struct be_ctrl_info *ctrl = &phba->ctrl;
3086
3087 /* Alloc MCC compl queue */
3088 cq = &phba->ctrl.mcc_obj.cq;
3089 if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3090 sizeof(struct be_mcc_compl)))
3091 goto err;
3092 /* Ask BE to create MCC compl queue; */
3093 if (phba->msix_enabled) {
3094 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3095 [phba->num_cpus].q, false, true, 0))
3096 goto mcc_cq_free;
3097 } else {
3098 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3099 false, true, 0))
3100 goto mcc_cq_free;
3101 }
3102
3103 /* Alloc MCC queue */
3104 q = &phba->ctrl.mcc_obj.q;
3105 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3106 goto mcc_cq_destroy;
3107
3108 /* Ask BE to create MCC queue */
35e66019 3109 if (beiscsi_cmd_mccq_create(phba, q, cq))
bfead3b2
JK		 3110 goto mcc_q_free;
3111
3112 return 0;
3113
3114mcc_q_free:
3115 be_queue_free(phba, q);
3116mcc_cq_destroy:
3117 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3118mcc_cq_free:
3119 be_queue_free(phba, cq);
3120err:
d3ad2bb3 3121 return -ENOMEM;
bfead3b2
JK		 3122}
3123
3124static int find_num_cpus(void)
3125{
3126 int num_cpus = 0;
3127
3128 num_cpus = num_online_cpus();
3129 if (num_cpus >= MAX_CPUS)
3130 num_cpus = MAX_CPUS - 1;
3131
457ff3b7 3132 SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus);
bfead3b2 3133 return num_cpus;
6733b39a
JK		 3134}
3135
3136static int hwi_init_port(struct beiscsi_hba *phba)
3137{
3138 struct hwi_controller *phwi_ctrlr;
3139 struct hwi_context_memory *phwi_context;
3140 unsigned int def_pdu_ring_sz;
3141 struct be_ctrl_info *ctrl = &phba->ctrl;
3142 int status;
3143
3144 def_pdu_ring_sz =
3145 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3146 phwi_ctrlr = phba->phwi_ctrlr;
6733b39a 3147 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 3148 phwi_context->max_eqd = 0;
3149 phwi_context->min_eqd = 0;
3150 phwi_context->cur_eqd = 64;
6733b39a 3151 be_cmd_fw_initialize(&phba->ctrl);
bfead3b2
JK		 3152
3153 status = beiscsi_create_eqs(phba, phwi_context);
6733b39a 3154 if (status != 0) {
457ff3b7 3155 shost_printk(KERN_ERR, phba->shost, "EQ not created\n");
6733b39a
JK		 3156 goto error;
3157 }
3158
bfead3b2
JK		 3159 status = be_mcc_queues_create(phba, phwi_context);
3160 if (status != 0)
3161 goto error;
3162
3163 status = mgmt_check_supported_fw(ctrl, phba);
6733b39a
JK		 3164 if (status != 0) {
3165 shost_printk(KERN_ERR, phba->shost,
457ff3b7 3166 "Unsupported fw version\n");
6733b39a
JK		 3167 goto error;
3168 }
3169
bfead3b2 3170 status = beiscsi_create_cqs(phba, phwi_context);
6733b39a
JK		 3171 if (status != 0) {
3172 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
3173 goto error;
3174 }
3175
3176 status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3177 def_pdu_ring_sz);
3178 if (status != 0) {
3179 shost_printk(KERN_ERR, phba->shost,
3180 "Default Header not created\n");
3181 goto error;
3182 }
3183
3184 status = beiscsi_create_def_data(phba, phwi_context,
3185 phwi_ctrlr, def_pdu_ring_sz);
3186 if (status != 0) {
3187 shost_printk(KERN_ERR, phba->shost,
3188 "Default Data not created\n");
3189 goto error;
3190 }
3191
3192 status = beiscsi_post_pages(phba);
3193 if (status != 0) {
3194 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
3195 goto error;
3196 }
3197
3198 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3199 if (status != 0) {
3200 shost_printk(KERN_ERR, phba->shost,
3201 "WRB Rings not created\n");
3202 goto error;
3203 }
3204
3205 SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
3206 return 0;
3207
3208error:
3209 shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
3210 hwi_cleanup(phba);
a49e06d5 3211 return status;
6733b39a
JK		 3212}
3213
6733b39a
JK		 3214static int hwi_init_controller(struct beiscsi_hba *phba)
3215{
3216 struct hwi_controller *phwi_ctrlr;
3217
3218 phwi_ctrlr = phba->phwi_ctrlr;
3219 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3220 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3221 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
457ff3b7 3222 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n",
6733b39a
JK		 3223 phwi_ctrlr->phwi_ctxt);
3224 } else {
3225 shost_printk(KERN_ERR, phba->shost,
3226 "HWI_MEM_ADDN_CONTEXT is more than one element."
3227 "Failing to load\n");
3228 return -ENOMEM;
3229 }
3230
3231 iscsi_init_global_templates(phba);
3ec78271
JK		 3232 if (beiscsi_init_wrb_handle(phba))
3233 return -ENOMEM;
3234
6733b39a
JK		 3235 hwi_init_async_pdu_ctx(phba);
3236 if (hwi_init_port(phba) != 0) {
3237 shost_printk(KERN_ERR, phba->shost,
3238 "hwi_init_controller failed\n");
3239 return -ENOMEM;
3240 }
3241 return 0;
3242}
3243
3244static void beiscsi_free_mem(struct beiscsi_hba *phba)
3245{
3246 struct be_mem_descriptor *mem_descr;
3247 int i, j;
3248
3249 mem_descr = phba->init_mem;
3250 i = 0;
3251 j = 0;
3252 for (i = 0; i < SE_MEM_MAX; i++) {
3253 for (j = mem_descr->num_elements; j > 0; j--) {
3254 pci_free_consistent(phba->pcidev,
3255 mem_descr->mem_array[j - 1].size,
3256 mem_descr->mem_array[j - 1].virtual_address,
457ff3b7
JK		 3257 (unsigned long)mem_descr->mem_array[j - 1].
3258 bus_address.u.a64.address);
6733b39a
JK		 3259 }
3260 kfree(mem_descr->mem_array);
3261 mem_descr++;
3262 }
3263 kfree(phba->init_mem);
3264 kfree(phba->phwi_ctrlr);
3265}
3266
3267static int beiscsi_init_controller(struct beiscsi_hba *phba)
3268{
3269 int ret = -ENOMEM;
3270
3271 ret = beiscsi_get_memory(phba);
3272 if (ret < 0) {
3273 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
457ff3b7 3274 "Failed in beiscsi_alloc_memory\n");
6733b39a
JK		 3275 return ret;
3276 }
3277
3278 ret = hwi_init_controller(phba);
3279 if (ret)
3280 goto free_init;
3281 SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
3282 return 0;
3283
3284free_init:
3285 beiscsi_free_mem(phba);
a49e06d5 3286 return ret;
6733b39a
JK		 3287}
3288
3289static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3290{
3291 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3292 struct sgl_handle *psgl_handle;
3293 struct iscsi_sge *pfrag;
3294 unsigned int arr_index, i, idx;
3295
3296 phba->io_sgl_hndl_avbl = 0;
3297 phba->eh_sgl_hndl_avbl = 0;
bfead3b2 3298
6733b39a
JK		 3299 mem_descr_sglh = phba->init_mem;
3300 mem_descr_sglh += HWI_MEM_SGLH;
3301 if (1 == mem_descr_sglh->num_elements) {
3302 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3303 phba->params.ios_per_ctrl,
3304 GFP_KERNEL);
3305 if (!phba->io_sgl_hndl_base) {
3306 shost_printk(KERN_ERR, phba->shost,
3307 "Mem Alloc Failed. Failing to load\n");
3308 return -ENOMEM;
3309 }
3310 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3311 (phba->params.icds_per_ctrl -
3312 phba->params.ios_per_ctrl),
3313 GFP_KERNEL);
3314 if (!phba->eh_sgl_hndl_base) {
3315 kfree(phba->io_sgl_hndl_base);
3316 shost_printk(KERN_ERR, phba->shost,
3317 "Mem Alloc Failed. Failing to load\n");
3318 return -ENOMEM;
3319 }
3320 } else {
3321 shost_printk(KERN_ERR, phba->shost,
3322 "HWI_MEM_SGLH is more than one element."
3323 "Failing to load\n");
3324 return -ENOMEM;
3325 }
3326
3327 arr_index = 0;
3328 idx = 0;
3329 while (idx < mem_descr_sglh->num_elements) {
3330 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3331
3332 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3333 sizeof(struct sgl_handle)); i++) {
3334 if (arr_index < phba->params.ios_per_ctrl) {
3335 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3336 phba->io_sgl_hndl_avbl++;
3337 arr_index++;
3338 } else {
3339 phba->eh_sgl_hndl_base[arr_index -
3340 phba->params.ios_per_ctrl] =
3341 psgl_handle;
3342 arr_index++;
3343 phba->eh_sgl_hndl_avbl++;
3344 }
3345 psgl_handle++;
3346 }
3347 idx++;
3348 }
3349 SE_DEBUG(DBG_LVL_8,
3350 "phba->io_sgl_hndl_avbl=%d"
457ff3b7 3351 "phba->eh_sgl_hndl_avbl=%d\n",
6733b39a
JK		 3352 phba->io_sgl_hndl_avbl,
3353 phba->eh_sgl_hndl_avbl);
3354 mem_descr_sg = phba->init_mem;
3355 mem_descr_sg += HWI_MEM_SGE;
457ff3b7 3356 SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n",
6733b39a
JK		 3357 mem_descr_sg->num_elements);
3358 arr_index = 0;
3359 idx = 0;
3360 while (idx < mem_descr_sg->num_elements) {
3361 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3362
3363 for (i = 0;
3364 i < (mem_descr_sg->mem_array[idx].size) /
3365 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3366 i++) {
3367 if (arr_index < phba->params.ios_per_ctrl)
3368 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3369 else
3370 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3371 phba->params.ios_per_ctrl];
3372 psgl_handle->pfrag = pfrag;
3373 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3374 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3375 pfrag += phba->params.num_sge_per_io;
3376 psgl_handle->sgl_index =
7da50879 3377 phba->fw_config.iscsi_icd_start + arr_index++;
6733b39a
JK		 3378 }
3379 idx++;
3380 }
3381 phba->io_sgl_free_index = 0;
3382 phba->io_sgl_alloc_index = 0;
3383 phba->eh_sgl_free_index = 0;
3384 phba->eh_sgl_alloc_index = 0;
3385 return 0;
3386}
3387
3388static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3389{
3390 int i, new_cid;
3391
c2462288 3392 phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
6733b39a
JK		 3393 GFP_KERNEL);
3394 if (!phba->cid_array) {
3395 shost_printk(KERN_ERR, phba->shost,
3396 "Failed to allocate memory in "
3397 "hba_setup_cid_tbls\n");
3398 return -ENOMEM;
3399 }
c2462288 3400 phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
6733b39a
JK		 3401 phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3402 if (!phba->ep_array) {
3403 shost_printk(KERN_ERR, phba->shost,
3404 "Failed to allocate memory in "
457ff3b7 3405 "hba_setup_cid_tbls\n");
6733b39a
JK		 3406 kfree(phba->cid_array);
3407 return -ENOMEM;
3408 }
7da50879 3409 new_cid = phba->fw_config.iscsi_cid_start;
6733b39a
JK		 3410 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3411 phba->cid_array[i] = new_cid;
3412 new_cid += 2;
3413 }
3414 phba->avlbl_cids = phba->params.cxns_per_ctrl;
3415 return 0;
3416}
3417
238f6b72 3418static void hwi_enable_intr(struct beiscsi_hba *phba)
6733b39a
JK		 3419{
3420 struct be_ctrl_info *ctrl = &phba->ctrl;
3421 struct hwi_controller *phwi_ctrlr;
3422 struct hwi_context_memory *phwi_context;
3423 struct be_queue_info *eq;
3424 u8 __iomem *addr;
bfead3b2 3425 u32 reg, i;
6733b39a
JK		 3426 u32 enabled;
3427
3428 phwi_ctrlr = phba->phwi_ctrlr;
3429 phwi_context = phwi_ctrlr->phwi_ctxt;
3430
6733b39a
JK		 3431 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3432 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3433 reg = ioread32(addr);
6733b39a
JK		 3434
3435 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3436 if (!enabled) {
3437 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
457ff3b7 3438 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr);
6733b39a 3439 iowrite32(reg, addr);
665d6d94
JK		 3440 }
3441
3442 if (!phba->msix_enabled) {
3443 eq = &phwi_context->be_eq[0].q;
3444 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3445 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3446 } else {
3447 for (i = 0; i <= phba->num_cpus; i++) {
3448 eq = &phwi_context->be_eq[i].q;
457ff3b7 3449 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
bfead3b2
JK		 3450 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3451 }
c03af1ae 3452 }
6733b39a
JK		 3453}
3454
3455static void hwi_disable_intr(struct beiscsi_hba *phba)
3456{
3457 struct be_ctrl_info *ctrl = &phba->ctrl;
3458
3459 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3460 u32 reg = ioread32(addr);
3461
3462 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3463 if (enabled) {
3464 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3465 iowrite32(reg, addr);
3466 } else
3467 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 3468 "In hwi_disable_intr, Already Disabled\n");
6733b39a
JK		 3469}
3470
c7acc5b8
JK		 3471static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3472{
3473 struct be_cmd_resp_get_boot_target *boot_resp;
3474 struct be_cmd_resp_get_session *session_resp;
3475 struct be_mcc_wrb *wrb;
3476 struct be_dma_mem nonemb_cmd;
3477 unsigned int tag, wrb_num;
3478 unsigned short status, extd_status;
3479 struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
f457a46f 3480 int ret = -ENOMEM;
c7acc5b8
JK		 3481
3482 tag = beiscsi_get_boot_target(phba);
3483 if (!tag) {
3484 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed\n");
3485 return -EAGAIN;
3486 } else
3487 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3488 phba->ctrl.mcc_numtag[tag]);
3489
3490 wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3491 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3492 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3493 if (status || extd_status) {
3494 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed"
3495 " status = %d extd_status = %d\n",
3496 status, extd_status);
3497 free_mcc_tag(&phba->ctrl, tag);
3498 return -EBUSY;
3499 }
3500 wrb = queue_get_wrb(mccq, wrb_num);
3501 free_mcc_tag(&phba->ctrl, tag);
3502 boot_resp = embedded_payload(wrb);
3503
3504 if (boot_resp->boot_session_handle < 0) {
f457a46f 3505 shost_printk(KERN_INFO, phba->shost, "No Boot Session.\n");
c7acc5b8
JK		 3506 return -ENXIO;
3507 }
3508
3509 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3510 sizeof(*session_resp),
3511 &nonemb_cmd.dma);
3512 if (nonemb_cmd.va == NULL) {
3513 SE_DEBUG(DBG_LVL_1,
3514 "Failed to allocate memory for"
3515 "beiscsi_get_session_info\n");
3516 return -ENOMEM;
3517 }
3518
3519 memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3520 tag = beiscsi_get_session_info(phba,
3521 boot_resp->boot_session_handle, &nonemb_cmd);
3522 if (!tag) {
3523 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info"
3524 " Failed\n");
3525 goto boot_freemem;
3526 } else
3527 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3528 phba->ctrl.mcc_numtag[tag]);
3529
3530 wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3531 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3532 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3533 if (status || extd_status) {
3534 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info Failed"
3535 " status = %d extd_status = %d\n",
3536 status, extd_status);
3537 free_mcc_tag(&phba->ctrl, tag);
3538 goto boot_freemem;
3539 }
3540 wrb = queue_get_wrb(mccq, wrb_num);
3541 free_mcc_tag(&phba->ctrl, tag);
3542 session_resp = nonemb_cmd.va ;
f457a46f 3543
c7acc5b8
JK		 3544 memcpy(&phba->boot_sess, &session_resp->session_info,
3545 sizeof(struct mgmt_session_info));
f457a46f
MC		 3546 ret = 0;
3547
c7acc5b8
JK		 3548boot_freemem:
3549 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3550 nonemb_cmd.va, nonemb_cmd.dma);
f457a46f
MC		 3551 return ret;
3552}
3553
3554static void beiscsi_boot_release(void *data)
3555{
3556 struct beiscsi_hba *phba = data;
3557
3558 scsi_host_put(phba->shost);
3559}
3560
3561static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3562{
3563 struct iscsi_boot_kobj *boot_kobj;
3564
3565 /* get boot info using mgmt cmd */
3566 if (beiscsi_get_boot_info(phba))
3567 /* Try to see if we can carry on without this */
3568 return 0;
3569
3570 phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3571 if (!phba->boot_kset)
3572 return -ENOMEM;
3573
3574 /* get a ref because the show function will ref the phba */
3575 if (!scsi_host_get(phba->shost))
3576 goto free_kset;
3577 boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3578 beiscsi_show_boot_tgt_info,
3579 beiscsi_tgt_get_attr_visibility,
3580 beiscsi_boot_release);
3581 if (!boot_kobj)
3582 goto put_shost;
3583
3584 if (!scsi_host_get(phba->shost))
3585 goto free_kset;
3586 boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3587 beiscsi_show_boot_ini_info,
3588 beiscsi_ini_get_attr_visibility,
3589 beiscsi_boot_release);
3590 if (!boot_kobj)
3591 goto put_shost;
3592
3593 if (!scsi_host_get(phba->shost))
3594 goto free_kset;
3595 boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3596 beiscsi_show_boot_eth_info,
3597 beiscsi_eth_get_attr_visibility,
3598 beiscsi_boot_release);
3599 if (!boot_kobj)
3600 goto put_shost;
3601 return 0;
3602
3603put_shost:
3604 scsi_host_put(phba->shost);
3605free_kset:
3606 iscsi_boot_destroy_kset(phba->boot_kset);
c7acc5b8
JK		 3607 return -ENOMEM;
3608}
3609
6733b39a
JK		 3610static int beiscsi_init_port(struct beiscsi_hba *phba)
3611{
3612 int ret;
3613
3614 ret = beiscsi_init_controller(phba);
3615 if (ret < 0) {
3616 shost_printk(KERN_ERR, phba->shost,
3617 "beiscsi_dev_probe - Failed in"
457ff3b7 3618 "beiscsi_init_controller\n");
6733b39a
JK		 3619 return ret;
3620 }
3621 ret = beiscsi_init_sgl_handle(phba);
3622 if (ret < 0) {
3623 shost_printk(KERN_ERR, phba->shost,
3624 "beiscsi_dev_probe - Failed in"
457ff3b7 3625 "beiscsi_init_sgl_handle\n");
6733b39a
JK		 3626 goto do_cleanup_ctrlr;
3627 }
3628
3629 if (hba_setup_cid_tbls(phba)) {
3630 shost_printk(KERN_ERR, phba->shost,
3631 "Failed in hba_setup_cid_tbls\n");
3632 kfree(phba->io_sgl_hndl_base);
3633 kfree(phba->eh_sgl_hndl_base);
3634 goto do_cleanup_ctrlr;
3635 }
3636
3637 return ret;
3638
3639do_cleanup_ctrlr:
3640 hwi_cleanup(phba);
3641 return ret;
3642}
3643
3644static void hwi_purge_eq(struct beiscsi_hba *phba)
3645{
3646 struct hwi_controller *phwi_ctrlr;
3647 struct hwi_context_memory *phwi_context;
3648 struct be_queue_info *eq;
3649 struct be_eq_entry *eqe = NULL;
bfead3b2 3650 int i, eq_msix;
756d29c8 3651 unsigned int num_processed;
6733b39a
JK		 3652
3653 phwi_ctrlr = phba->phwi_ctrlr;
3654 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 3655 if (phba->msix_enabled)
3656 eq_msix = 1;
3657 else
3658 eq_msix = 0;
6733b39a 3659
bfead3b2
JK		 3660 for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3661 eq = &phwi_context->be_eq[i].q;
6733b39a 3662 eqe = queue_tail_node(eq);
756d29c8 3663 num_processed = 0;
bfead3b2
JK		 3664 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3665 & EQE_VALID_MASK) {
3666 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3667 queue_tail_inc(eq);
3668 eqe = queue_tail_node(eq);
756d29c8 3669 num_processed++;
bfead3b2 3670 }
756d29c8
JK		 3671
3672 if (num_processed)
3673 hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
6733b39a
JK		 3674 }
3675}
3676
3677static void beiscsi_clean_port(struct beiscsi_hba *phba)
3678{
03a12310 3679 int mgmt_status;
6733b39a
JK		 3680
3681 mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3682 if (mgmt_status)
3683 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 3684 "mgmt_epfw_cleanup FAILED\n");
756d29c8 3685
6733b39a 3686 hwi_purge_eq(phba);
756d29c8 3687 hwi_cleanup(phba);
6733b39a
JK		 3688 kfree(phba->io_sgl_hndl_base);
3689 kfree(phba->eh_sgl_hndl_base);
3690 kfree(phba->cid_array);
3691 kfree(phba->ep_array);
3692}
3693
1282ab76
MC		 3694static void beiscsi_cleanup_task(struct iscsi_task *task)
3695{
3696 struct beiscsi_io_task *io_task = task->dd_data;
3697 struct iscsi_conn *conn = task->conn;
3698 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3699 struct beiscsi_hba *phba = beiscsi_conn->phba;
3700 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3701 struct hwi_wrb_context *pwrb_context;
3702 struct hwi_controller *phwi_ctrlr;
3703
3704 phwi_ctrlr = phba->phwi_ctrlr;
3705 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3706 - phba->fw_config.iscsi_cid_start];
3707
3708 if (io_task->cmd_bhs) {
3709 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3710 io_task->bhs_pa.u.a64.address);
3711 io_task->cmd_bhs = NULL;
3712 }
3713
3714 if (task->sc) {
3715 if (io_task->pwrb_handle) {
3716 free_wrb_handle(phba, pwrb_context,
3717 io_task->pwrb_handle);
3718 io_task->pwrb_handle = NULL;
3719 }
3720
3721 if (io_task->psgl_handle) {
3722 spin_lock(&phba->io_sgl_lock);
3723 free_io_sgl_handle(phba, io_task->psgl_handle);
3724 spin_unlock(&phba->io_sgl_lock);
3725 io_task->psgl_handle = NULL;
3726 }
3727 } else {
3728 if (!beiscsi_conn->login_in_progress) {
3729 if (io_task->pwrb_handle) {
3730 free_wrb_handle(phba, pwrb_context,
3731 io_task->pwrb_handle);
3732 io_task->pwrb_handle = NULL;
3733 }
3734 if (io_task->psgl_handle) {
3735 spin_lock(&phba->mgmt_sgl_lock);
3736 free_mgmt_sgl_handle(phba,
3737 io_task->psgl_handle);
3738 spin_unlock(&phba->mgmt_sgl_lock);
3739 io_task->psgl_handle = NULL;
3740 }
3741 }
3742 }
3743}
3744
6733b39a
JK		 3745void
3746beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3747 struct beiscsi_offload_params *params)
3748{
3749 struct wrb_handle *pwrb_handle;
3750 struct iscsi_target_context_update_wrb *pwrb = NULL;
3751 struct be_mem_descriptor *mem_descr;
3752 struct beiscsi_hba *phba = beiscsi_conn->phba;
1282ab76
MC		 3753 struct iscsi_task *task = beiscsi_conn->task;
3754 struct iscsi_session *session = task->conn->session;
6733b39a
JK		 3755 u32 doorbell = 0;
3756
3757 /*
3758 * We can always use 0 here because it is reserved by libiscsi for
3759 * login/startup related tasks.
3760 */
1282ab76
MC		 3761 beiscsi_conn->login_in_progress = 0;
3762 spin_lock_bh(&session->lock);
3763 beiscsi_cleanup_task(task);
3764 spin_unlock_bh(&session->lock);
3765
7da50879 3766 pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
d5431488 3767 phba->fw_config.iscsi_cid_start));
6733b39a
JK		 3768 pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3769 memset(pwrb, 0, sizeof(*pwrb));
3770 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3771 max_burst_length, pwrb, params->dw[offsetof
3772 (struct amap_beiscsi_offload_params,
3773 max_burst_length) / 32]);
3774 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3775 max_send_data_segment_length, pwrb,
3776 params->dw[offsetof(struct amap_beiscsi_offload_params,
3777 max_send_data_segment_length) / 32]);
3778 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3779 first_burst_length,
3780 pwrb,
3781 params->dw[offsetof(struct amap_beiscsi_offload_params,
3782 first_burst_length) / 32]);
3783
3784 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3785 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3786 erl) / 32] & OFFLD_PARAMS_ERL));
3787 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3788 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3789 dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3790 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3791 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3792 hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3793 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3794 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3795 ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3796 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3797 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3798 imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3799 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3800 pwrb,
3801 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3802 exp_statsn) / 32] + 1));
3803 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3804 0x7);
3805 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3806 pwrb, pwrb_handle->wrb_index);
3807 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3808 pwrb, pwrb_handle->nxt_wrb_index);
3809 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3810 session_state, pwrb, 0);
3811 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3812 pwrb, 1);
3813 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3814 pwrb, 0);
3815 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3816 0);
3817
3818 mem_descr = phba->init_mem;
3819 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3820
3821 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3822 pad_buffer_addr_hi, pwrb,
3823 mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3824 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3825 pad_buffer_addr_lo, pwrb,
3826 mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3827
3828 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3829
3830 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
32951dd8 3831 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
bfead3b2 3832 << DB_DEF_PDU_WRB_INDEX_SHIFT;
6733b39a
JK		 3833 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3834
3835 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3836}
3837
3838static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3839 int *index, int *age)
3840{
bfead3b2 3841 *index = (int)itt;
6733b39a
JK		 3842 if (age)
3843 *age = conn->session->age;
3844}
3845
3846/**
3847 * beiscsi_alloc_pdu - allocates pdu and related resources
3848 * @task: libiscsi task
3849 * @opcode: opcode of pdu for task
3850 *
3851 * This is called with the session lock held. It will allocate
3852 * the wrb and sgl if needed for the command. And it will prep
3853 * the pdu's itt. beiscsi_parse_pdu will later translate
3854 * the pdu itt to the libiscsi task itt.
3855 */
3856static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3857{
3858 struct beiscsi_io_task *io_task = task->dd_data;
3859 struct iscsi_conn *conn = task->conn;
3860 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3861 struct beiscsi_hba *phba = beiscsi_conn->phba;
3862 struct hwi_wrb_context *pwrb_context;
3863 struct hwi_controller *phwi_ctrlr;
3864 itt_t itt;
2afc95bf
JK		 3865 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3866 dma_addr_t paddr;
6733b39a 3867
2afc95bf 3868 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
bc7accec 3869 GFP_ATOMIC, &paddr);
2afc95bf
JK		 3870 if (!io_task->cmd_bhs)
3871 return -ENOMEM;
2afc95bf 3872 io_task->bhs_pa.u.a64.address = paddr;
bfead3b2 3873 io_task->libiscsi_itt = (itt_t)task->itt;
6733b39a
JK		 3874 io_task->conn = beiscsi_conn;
3875
3876 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3877 task->hdr_max = sizeof(struct be_cmd_bhs);
d2cecf0d 3878 io_task->psgl_handle = NULL;
3ec78271 3879 io_task->pwrb_handle = NULL;
6733b39a
JK		 3880
3881 if (task->sc) {
3882 spin_lock(&phba->io_sgl_lock);
3883 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3884 spin_unlock(&phba->io_sgl_lock);
2afc95bf
JK		 3885 if (!io_task->psgl_handle)
3886 goto free_hndls;
d2cecf0d
JK		 3887 io_task->pwrb_handle = alloc_wrb_handle(phba,
3888 beiscsi_conn->beiscsi_conn_cid -
3889 phba->fw_config.iscsi_cid_start);
3890 if (!io_task->pwrb_handle)
3891 goto free_io_hndls;
6733b39a
JK		 3892 } else {
3893 io_task->scsi_cmnd = NULL;
d7aea67b 3894 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
6733b39a
JK		 3895 if (!beiscsi_conn->login_in_progress) {
3896 spin_lock(&phba->mgmt_sgl_lock);
3897 io_task->psgl_handle = (struct sgl_handle *)
3898 alloc_mgmt_sgl_handle(phba);
3899 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 3900 if (!io_task->psgl_handle)
3901 goto free_hndls;
3902
6733b39a
JK		 3903 beiscsi_conn->login_in_progress = 1;
3904 beiscsi_conn->plogin_sgl_handle =
3905 io_task->psgl_handle;
d2cecf0d
JK		 3906 io_task->pwrb_handle =
3907 alloc_wrb_handle(phba,
3908 beiscsi_conn->beiscsi_conn_cid -
3909 phba->fw_config.iscsi_cid_start);
3910 if (!io_task->pwrb_handle)
3911 goto free_io_hndls;
3912 beiscsi_conn->plogin_wrb_handle =
3913 io_task->pwrb_handle;
3914
6733b39a
JK		 3915 } else {
3916 io_task->psgl_handle =
3917 beiscsi_conn->plogin_sgl_handle;
d2cecf0d
JK		 3918 io_task->pwrb_handle =
3919 beiscsi_conn->plogin_wrb_handle;
6733b39a 3920 }
1282ab76 3921 beiscsi_conn->task = task;
6733b39a
JK		 3922 } else {
3923 spin_lock(&phba->mgmt_sgl_lock);
3924 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3925 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 3926 if (!io_task->psgl_handle)
3927 goto free_hndls;
d2cecf0d
JK		 3928 io_task->pwrb_handle =
3929 alloc_wrb_handle(phba,
3930 beiscsi_conn->beiscsi_conn_cid -
3931 phba->fw_config.iscsi_cid_start);
3932 if (!io_task->pwrb_handle)
3933 goto free_mgmt_hndls;
3934
6733b39a
JK		 3935 }
3936 }
bfead3b2
JK		 3937 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3938 wrb_index << 16) | (unsigned int)
3939 (io_task->psgl_handle->sgl_index));
32951dd8 3940 io_task->pwrb_handle->pio_handle = task;
bfead3b2 3941
6733b39a
JK		 3942 io_task->cmd_bhs->iscsi_hdr.itt = itt;
3943 return 0;
2afc95bf 3944
d2cecf0d
JK		 3945free_io_hndls:
3946 spin_lock(&phba->io_sgl_lock);
3947 free_io_sgl_handle(phba, io_task->psgl_handle);
3948 spin_unlock(&phba->io_sgl_lock);
3949 goto free_hndls;
3950free_mgmt_hndls:
3951 spin_lock(&phba->mgmt_sgl_lock);
3952 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3953 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 3954free_hndls:
3955 phwi_ctrlr = phba->phwi_ctrlr;
7da50879
JK		 3956 pwrb_context = &phwi_ctrlr->wrb_context[
3957 beiscsi_conn->beiscsi_conn_cid -
3958 phba->fw_config.iscsi_cid_start];
d2cecf0d
JK		 3959 if (io_task->pwrb_handle)
3960 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2afc95bf
JK		 3961 io_task->pwrb_handle = NULL;
3962 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3963 io_task->bhs_pa.u.a64.address);
1282ab76 3964 io_task->cmd_bhs = NULL;
457ff3b7 3965 SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n");
2afc95bf 3966 return -ENOMEM;
6733b39a
JK		 3967}
3968
6733b39a
JK		 3969static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3970 unsigned int num_sg, unsigned int xferlen,
3971 unsigned int writedir)
3972{
3973
3974 struct beiscsi_io_task *io_task = task->dd_data;
3975 struct iscsi_conn *conn = task->conn;
3976 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3977 struct beiscsi_hba *phba = beiscsi_conn->phba;
3978 struct iscsi_wrb *pwrb = NULL;
3979 unsigned int doorbell = 0;
3980
3981 pwrb = io_task->pwrb_handle->pwrb;
6733b39a
JK		 3982 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3983 io_task->bhs_len = sizeof(struct be_cmd_bhs);
3984
3985 if (writedir) {
6733b39a
JK		 3986 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3987 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3988 &io_task->cmd_bhs->iscsi_data_pdu,
3989 (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3990 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3991 &io_task->cmd_bhs->iscsi_data_pdu,
3992 ISCSI_OPCODE_SCSI_DATA_OUT);
3993 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3994 &io_task->cmd_bhs->iscsi_data_pdu, 1);
32951dd8
JK		 3995 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3996 INI_WR_CMD);
6733b39a 3997 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
6733b39a 3998 } else {
32951dd8
JK		 3999 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4000 INI_RD_CMD);
6733b39a
JK		 4001 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4002 }
4003 memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
4004 dw[offsetof(struct amap_pdu_data_out, lun) / 32],
516f43a2 4005 &io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
6733b39a
JK		 4006
4007 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
dc63aac6
JK		 4008 cpu_to_be16(*(unsigned short *)
4009 &io_task->cmd_bhs->iscsi_hdr.lun));
6733b39a
JK		 4010 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4011 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4012 io_task->pwrb_handle->wrb_index);
4013 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4014 be32_to_cpu(task->cmdsn));
4015 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4016 io_task->psgl_handle->sgl_index);
4017
4018 hwi_write_sgl(pwrb, sg, num_sg, io_task);
4019
4020 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4021 io_task->pwrb_handle->nxt_wrb_index);
4022 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4023
4024 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
32951dd8 4025 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 4026 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4027 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4028
4029 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4030 return 0;
4031}
4032
4033static int beiscsi_mtask(struct iscsi_task *task)
4034{
dafab8e0 4035 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 4036 struct iscsi_conn *conn = task->conn;
4037 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4038 struct beiscsi_hba *phba = beiscsi_conn->phba;
4039 struct iscsi_wrb *pwrb = NULL;
4040 unsigned int doorbell = 0;
dafab8e0 4041 unsigned int cid;
6733b39a 4042
bfead3b2 4043 cid = beiscsi_conn->beiscsi_conn_cid;
6733b39a 4044 pwrb = io_task->pwrb_handle->pwrb;
caf818f1 4045 memset(pwrb, 0, sizeof(*pwrb));
6733b39a
JK		 4046 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4047 be32_to_cpu(task->cmdsn));
4048 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4049 io_task->pwrb_handle->wrb_index);
4050 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4051 io_task->psgl_handle->sgl_index);
dafab8e0 4052
6733b39a
JK		 4053 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4054 case ISCSI_OP_LOGIN:
32951dd8
JK		 4055 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4056 TGT_DM_CMD);
6733b39a
JK		 4057 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4058 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4059 hwi_write_buffer(pwrb, task);
4060 break;
4061 case ISCSI_OP_NOOP_OUT:
1390b01b
JK		 4062 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4063 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4064 TGT_DM_CMD);
4065 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt,
4066 pwrb, 0);
685e16fd 4067 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
1390b01b
JK		 4068 } else {
4069 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4070 INI_RD_CMD);
685e16fd 4071 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
1390b01b 4072 }
6733b39a
JK		 4073 hwi_write_buffer(pwrb, task);
4074 break;
4075 case ISCSI_OP_TEXT:
32951dd8 4076 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
b30c6dab 4077 TGT_DM_CMD);
0ecb0b45 4078 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
6733b39a
JK		 4079 hwi_write_buffer(pwrb, task);
4080 break;
4081 case ISCSI_OP_SCSI_TMFUNC:
32951dd8
JK		 4082 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4083 INI_TMF_CMD);
6733b39a
JK		 4084 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4085 hwi_write_buffer(pwrb, task);
4086 break;
4087 case ISCSI_OP_LOGOUT:
4088 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4089 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
dafab8e0 4090 HWH_TYPE_LOGOUT);
6733b39a
JK		 4091 hwi_write_buffer(pwrb, task);
4092 break;
4093
4094 default:
457ff3b7 4095 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n",
6733b39a
JK		 4096 task->hdr->opcode & ISCSI_OPCODE_MASK);
4097 return -EINVAL;
4098 }
4099
4100 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
51a46250 4101 task->data_count);
6733b39a
JK		 4102 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4103 io_task->pwrb_handle->nxt_wrb_index);
4104 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4105
bfead3b2 4106 doorbell |= cid & DB_WRB_POST_CID_MASK;
32951dd8 4107 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 4108 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4109 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4110 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4111 return 0;
4112}
4113
4114static int beiscsi_task_xmit(struct iscsi_task *task)
4115{
6733b39a
JK		 4116 struct beiscsi_io_task *io_task = task->dd_data;
4117 struct scsi_cmnd *sc = task->sc;
6733b39a
JK		 4118 struct scatterlist *sg;
4119 int num_sg;
4120 unsigned int writedir = 0, xferlen = 0;
4121
6733b39a
JK		 4122 if (!sc)
4123 return beiscsi_mtask(task);
4124
4125 io_task->scsi_cmnd = sc;
4126 num_sg = scsi_dma_map(sc);
4127 if (num_sg < 0) {
4128 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
4129 return num_sg;
4130 }
6733b39a
JK		 4131 xferlen = scsi_bufflen(sc);
4132 sg = scsi_sglist(sc);
4133 if (sc->sc_data_direction == DMA_TO_DEVICE) {
4134 writedir = 1;
457ff3b7 4135 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n",
6733b39a
JK		 4136 task->imm_count);
4137 } else
4138 writedir = 0;
4139 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
4140}
4141
25602c97 4142static void beiscsi_quiesce(struct beiscsi_hba *phba)
6733b39a 4143{
bfead3b2
JK		 4144 struct hwi_controller *phwi_ctrlr;
4145 struct hwi_context_memory *phwi_context;
4146 struct be_eq_obj *pbe_eq;
4147 unsigned int i, msix_vec;
e9b91193
JK		 4148 u8 *real_offset = 0;
4149 u32 value = 0;
6733b39a 4150
bfead3b2
JK		 4151 phwi_ctrlr = phba->phwi_ctrlr;
4152 phwi_context = phwi_ctrlr->phwi_ctxt;
6733b39a 4153 hwi_disable_intr(phba);
bfead3b2
JK		 4154 if (phba->msix_enabled) {
4155 for (i = 0; i <= phba->num_cpus; i++) {
4156 msix_vec = phba->msix_entries[i].vector;
4157 free_irq(msix_vec, &phwi_context->be_eq[i]);
8fcfb210 4158 kfree(phba->msi_name[i]);
bfead3b2
JK		 4159 }
4160 } else
4161 if (phba->pcidev->irq)
4162 free_irq(phba->pcidev->irq, phba);
4163 pci_disable_msix(phba->pcidev);
6733b39a
JK		 4164 destroy_workqueue(phba->wq);
4165 if (blk_iopoll_enabled)
bfead3b2
JK		 4166 for (i = 0; i < phba->num_cpus; i++) {
4167 pbe_eq = &phwi_context->be_eq[i];
4168 blk_iopoll_disable(&pbe_eq->iopoll);
4169 }
6733b39a
JK		 4170
4171 beiscsi_clean_port(phba);
4172 beiscsi_free_mem(phba);
e9b91193
JK		 4173 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4174
4175 value = readl((void *)real_offset);
4176
4177 if (value & 0x00010000) {
4178 value &= 0xfffeffff;
4179 writel(value, (void *)real_offset);
4180 }
6733b39a
JK		 4181 beiscsi_unmap_pci_function(phba);
4182 pci_free_consistent(phba->pcidev,
4183 phba->ctrl.mbox_mem_alloced.size,
4184 phba->ctrl.mbox_mem_alloced.va,
4185 phba->ctrl.mbox_mem_alloced.dma);
25602c97
JK		 4186}
4187
4188static void beiscsi_remove(struct pci_dev *pcidev)
4189{
4190
4191 struct beiscsi_hba *phba = NULL;
4192
4193 phba = pci_get_drvdata(pcidev);
4194 if (!phba) {
4195 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4196 return;
4197 }
4198
4199 beiscsi_quiesce(phba);
9d045163 4200 iscsi_boot_destroy_kset(phba->boot_kset);
6733b39a
JK		 4201 iscsi_host_remove(phba->shost);
4202 pci_dev_put(phba->pcidev);
4203 iscsi_host_free(phba->shost);
8dce69ff 4204 pci_disable_device(pcidev);
6733b39a
JK		 4205}
4206
25602c97
JK		 4207static void beiscsi_shutdown(struct pci_dev *pcidev)
4208{
4209
4210 struct beiscsi_hba *phba = NULL;
4211
4212 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4213 if (!phba) {
4214 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4215 return;
4216 }
4217
4218 beiscsi_quiesce(phba);
8dce69ff 4219 pci_disable_device(pcidev);
25602c97
JK		 4220}
4221
bfead3b2
JK		 4222static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4223{
4224 int i, status;
4225
4226 for (i = 0; i <= phba->num_cpus; i++)
4227 phba->msix_entries[i].entry = i;
4228
4229 status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4230 (phba->num_cpus + 1));
4231 if (!status)
4232 phba->msix_enabled = true;
4233
4234 return;
4235}
4236
6733b39a
JK		 4237static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
4238 const struct pci_device_id *id)
4239{
4240 struct beiscsi_hba *phba = NULL;
bfead3b2
JK		 4241 struct hwi_controller *phwi_ctrlr;
4242 struct hwi_context_memory *phwi_context;
4243 struct be_eq_obj *pbe_eq;
238f6b72 4244 int ret, num_cpus, i;
e9b91193
JK		 4245 u8 *real_offset = 0;
4246 u32 value = 0;
6733b39a
JK		 4247
4248 ret = beiscsi_enable_pci(pcidev);
4249 if (ret < 0) {
82284c09
DC		 4250 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4251 " Failed to enable pci device\n");
6733b39a
JK		 4252 return ret;
4253 }
4254
4255 phba = beiscsi_hba_alloc(pcidev);
4256 if (!phba) {
4257 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
457ff3b7 4258 " Failed in beiscsi_hba_alloc\n");
6733b39a
JK		 4259 goto disable_pci;
4260 }
4261
f98c96b0
JK		 4262 switch (pcidev->device) {
4263 case BE_DEVICE_ID1:
4264 case OC_DEVICE_ID1:
4265 case OC_DEVICE_ID2:
4266 phba->generation = BE_GEN2;
4267 break;
4268 case BE_DEVICE_ID2:
4269 case OC_DEVICE_ID3:
4270 phba->generation = BE_GEN3;
4271 break;
4272 default:
4273 phba->generation = 0;
4274 }
4275
bfead3b2
JK		 4276 if (enable_msix)
4277 num_cpus = find_num_cpus();
4278 else
4279 num_cpus = 1;
4280 phba->num_cpus = num_cpus;
457ff3b7 4281 SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus);
bfead3b2 4282
b547f2d6 4283 if (enable_msix) {
bfead3b2 4284 beiscsi_msix_enable(phba);
b547f2d6
JK		 4285 if (!phba->msix_enabled)
4286 phba->num_cpus = 1;
4287 }
6733b39a
JK		 4288 ret = be_ctrl_init(phba, pcidev);
4289 if (ret) {
4290 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4291 "Failed in be_ctrl_init\n");
4292 goto hba_free;
4293 }
4294
e9b91193
JK		 4295 if (!num_hba) {
4296 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4297 value = readl((void *)real_offset);
4298 if (value & 0x00010000) {
4299 gcrashmode++;
4300 shost_printk(KERN_ERR, phba->shost,
4301 "Loading Driver in crashdump mode\n");
e5285860 4302 ret = beiscsi_cmd_reset_function(phba);
e9b91193
JK		 4303 if (ret) {
4304 shost_printk(KERN_ERR, phba->shost,
4305 "Reset Failed. Aborting Crashdump\n");
4306 goto hba_free;
4307 }
4308 ret = be_chk_reset_complete(phba);
4309 if (ret) {
4310 shost_printk(KERN_ERR, phba->shost,
4311 "Failed to get out of reset."
4312 "Aborting Crashdump\n");
4313 goto hba_free;
4314 }
4315 } else {
4316 value |= 0x00010000;
4317 writel(value, (void *)real_offset);
4318 num_hba++;
4319 }
4320 }
4321
6733b39a
JK		 4322 spin_lock_init(&phba->io_sgl_lock);
4323 spin_lock_init(&phba->mgmt_sgl_lock);
4324 spin_lock_init(&phba->isr_lock);
7da50879
JK		 4325 ret = mgmt_get_fw_config(&phba->ctrl, phba);
4326 if (ret != 0) {
4327 shost_printk(KERN_ERR, phba->shost,
4328 "Error getting fw config\n");
4329 goto free_port;
4330 }
4331 phba->shost->max_id = phba->fw_config.iscsi_cid_count;
6733b39a 4332 beiscsi_get_params(phba);
aa874f07 4333 phba->shost->can_queue = phba->params.ios_per_ctrl;
6733b39a
JK		 4334 ret = beiscsi_init_port(phba);
4335 if (ret < 0) {
4336 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4337 "Failed in beiscsi_init_port\n");
4338 goto free_port;
4339 }
4340
756d29c8
JK		 4341 for (i = 0; i < MAX_MCC_CMD ; i++) {
4342 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4343 phba->ctrl.mcc_tag[i] = i + 1;
4344 phba->ctrl.mcc_numtag[i + 1] = 0;
4345 phba->ctrl.mcc_tag_available++;
4346 }
4347
4348 phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4349
6733b39a
JK		 4350 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
4351 phba->shost->host_no);
278274d5 4352 phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
6733b39a
JK		 4353 if (!phba->wq) {
4354 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4355 "Failed to allocate work queue\n");
4356 goto free_twq;
4357 }
4358
4359 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
4360
bfead3b2
JK		 4361 phwi_ctrlr = phba->phwi_ctrlr;
4362 phwi_context = phwi_ctrlr->phwi_ctxt;
6733b39a 4363 if (blk_iopoll_enabled) {
bfead3b2
JK		 4364 for (i = 0; i < phba->num_cpus; i++) {
4365 pbe_eq = &phwi_context->be_eq[i];
4366 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4367 be_iopoll);
4368 blk_iopoll_enable(&pbe_eq->iopoll);
4369 }
6733b39a 4370 }
6733b39a
JK		 4371 ret = beiscsi_init_irqs(phba);
4372 if (ret < 0) {
4373 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4374 "Failed to beiscsi_init_irqs\n");
4375 goto free_blkenbld;
4376 }
238f6b72 4377 hwi_enable_intr(phba);
f457a46f
MC		 4378
4379 if (beiscsi_setup_boot_info(phba))
4380 /*
4381 * log error but continue, because we may not be using
4382 * iscsi boot.
4383 */
4384 shost_printk(KERN_ERR, phba->shost, "Could not set up "
4385 "iSCSI boot info.");
4386
457ff3b7 4387 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
6733b39a
JK		 4388 return 0;
4389
6733b39a
JK		 4390free_blkenbld:
4391 destroy_workqueue(phba->wq);
4392 if (blk_iopoll_enabled)
bfead3b2
JK		 4393 for (i = 0; i < phba->num_cpus; i++) {
4394 pbe_eq = &phwi_context->be_eq[i];
4395 blk_iopoll_disable(&pbe_eq->iopoll);
4396 }
6733b39a
JK		 4397free_twq:
4398 beiscsi_clean_port(phba);
4399 beiscsi_free_mem(phba);
4400free_port:
e9b91193
JK		 4401 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4402
4403 value = readl((void *)real_offset);
4404
4405 if (value & 0x00010000) {
4406 value &= 0xfffeffff;
4407 writel(value, (void *)real_offset);
4408 }
4409
6733b39a
JK		 4410 pci_free_consistent(phba->pcidev,
4411 phba->ctrl.mbox_mem_alloced.size,
4412 phba->ctrl.mbox_mem_alloced.va,
4413 phba->ctrl.mbox_mem_alloced.dma);
4414 beiscsi_unmap_pci_function(phba);
4415hba_free:
238f6b72
JK		 4416 if (phba->msix_enabled)
4417 pci_disable_msix(phba->pcidev);
6733b39a
JK		 4418 iscsi_host_remove(phba->shost);
4419 pci_dev_put(phba->pcidev);
4420 iscsi_host_free(phba->shost);
4421disable_pci:
4422 pci_disable_device(pcidev);
4423 return ret;
4424}
4425
4426struct iscsi_transport beiscsi_iscsi_transport = {
4427 .owner = THIS_MODULE,
4428 .name = DRV_NAME,
9db0fb3a 4429 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
6733b39a 4430 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
6733b39a
JK		 4431 .create_session = beiscsi_session_create,
4432 .destroy_session = beiscsi_session_destroy,
4433 .create_conn = beiscsi_conn_create,
4434 .bind_conn = beiscsi_conn_bind,
4435 .destroy_conn = iscsi_conn_teardown,
3128c6c7 4436 .attr_is_visible = be2iscsi_attr_is_visible,
6733b39a 4437 .set_param = beiscsi_set_param,
c7f7fd5b 4438 .get_conn_param = iscsi_conn_get_param,
6733b39a
JK		 4439 .get_session_param = iscsi_session_get_param,
4440 .get_host_param = beiscsi_get_host_param,
4441 .start_conn = beiscsi_conn_start,
fa95d206 4442 .stop_conn = iscsi_conn_stop,
6733b39a
JK		 4443 .send_pdu = iscsi_conn_send_pdu,
4444 .xmit_task = beiscsi_task_xmit,
4445 .cleanup_task = beiscsi_cleanup_task,
4446 .alloc_pdu = beiscsi_alloc_pdu,
4447 .parse_pdu_itt = beiscsi_parse_pdu,
4448 .get_stats = beiscsi_conn_get_stats,
c7f7fd5b 4449 .get_ep_param = beiscsi_ep_get_param,
6733b39a
JK		 4450 .ep_connect = beiscsi_ep_connect,
4451 .ep_poll = beiscsi_ep_poll,
4452 .ep_disconnect = beiscsi_ep_disconnect,
4453 .session_recovery_timedout = iscsi_session_recovery_timedout,
4454};
4455
4456static struct pci_driver beiscsi_pci_driver = {
4457 .name = DRV_NAME,
4458 .probe = beiscsi_dev_probe,
4459 .remove = beiscsi_remove,
25602c97 4460 .shutdown = beiscsi_shutdown,
6733b39a
JK		 4461 .id_table = beiscsi_pci_id_table
4462};
4463
bfead3b2 4464
6733b39a
JK		 4465static int __init beiscsi_module_init(void)
4466{
4467 int ret;
4468
4469 beiscsi_scsi_transport =
4470 iscsi_register_transport(&beiscsi_iscsi_transport);
4471 if (!beiscsi_scsi_transport) {
4472 SE_DEBUG(DBG_LVL_1,
4473 "beiscsi_module_init - Unable to register beiscsi"
4474 "transport.\n");
f55a24f2 4475 return -ENOMEM;
6733b39a 4476 }
457ff3b7 4477 SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n",
6733b39a
JK		 4478 &beiscsi_iscsi_transport);
4479
4480 ret = pci_register_driver(&beiscsi_pci_driver);
4481 if (ret) {
4482 SE_DEBUG(DBG_LVL_1,
4483 "beiscsi_module_init - Unable to register"
4484 "beiscsi pci driver.\n");
4485 goto unregister_iscsi_transport;
4486 }
4487 return 0;
4488
4489unregister_iscsi_transport:
4490 iscsi_unregister_transport(&beiscsi_iscsi_transport);
4491 return ret;
4492}
4493
4494static void __exit beiscsi_module_exit(void)
4495{
4496 pci_unregister_driver(&beiscsi_pci_driver);
4497 iscsi_unregister_transport(&beiscsi_iscsi_transport);
4498}
4499
4500module_init(beiscsi_module_init);
4501module_exit(beiscsi_module_exit);
ubuntu-bionic-kernel mirror