2 * This file is part of the Emulex Linux Device Driver for Enterprise iSCSI
3 * Host Bus Adapters. Refer to the README file included with this package
4 * for driver version and adapter compatibility.
6 * Copyright (c) 2018 Broadcom. All Rights Reserved.
7 * The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of version 2 of the GNU General Public License as published
11 * by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful. ALL EXPRESS
14 * OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, INCLUDING ANY
15 * IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
16 * OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH
17 * DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
18 * See the GNU General Public License for more details, a copy of which
19 * can be found in the file COPYING included with this package.
21 * Contact Information:
22 * linux-drivers@broadcom.com
26 #include <linux/reboot.h>
27 #include <linux/delay.h>
28 #include <linux/slab.h>
29 #include <linux/interrupt.h>
30 #include <linux/blkdev.h>
31 #include <linux/pci.h>
32 #include <linux/string.h>
33 #include <linux/kernel.h>
34 #include <linux/semaphore.h>
35 #include <linux/iscsi_boot_sysfs.h>
36 #include <linux/module.h>
37 #include <linux/bsg-lib.h>
38 #include <linux/irq_poll.h>
40 #include <scsi/libiscsi.h>
41 #include <scsi/scsi_bsg_iscsi.h>
42 #include <scsi/scsi_netlink.h>
43 #include <scsi/scsi_transport_iscsi.h>
44 #include <scsi/scsi_transport.h>
45 #include <scsi/scsi_cmnd.h>
46 #include <scsi/scsi_device.h>
47 #include <scsi/scsi_host.h>
48 #include <scsi/scsi.h>
54 static unsigned int be_iopoll_budget
= 10;
55 static unsigned int be_max_phys_size
= 64;
56 static unsigned int enable_msix
= 1;
58 MODULE_DESCRIPTION(DRV_DESC
" " BUILD_STR
);
59 MODULE_VERSION(BUILD_STR
);
60 MODULE_AUTHOR("Emulex Corporation");
61 MODULE_LICENSE("GPL");
62 module_param(be_iopoll_budget
, int, 0);
63 module_param(enable_msix
, int, 0);
64 module_param(be_max_phys_size
, uint
, S_IRUGO
);
65 MODULE_PARM_DESC(be_max_phys_size
,
66 "Maximum Size (In Kilobytes) of physically contiguous "
67 "memory that can be allocated. Range is 16 - 128");
69 #define beiscsi_disp_param(_name)\
71 beiscsi_##_name##_disp(struct device *dev,\
72 struct device_attribute *attrib, char *buf) \
74 struct Scsi_Host *shost = class_to_shost(dev);\
75 struct beiscsi_hba *phba = iscsi_host_priv(shost); \
76 return snprintf(buf, PAGE_SIZE, "%d\n",\
80 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
82 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
84 if (val >= _minval && val <= _maxval) {\
85 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
86 "BA_%d : beiscsi_"#_name" updated "\
87 "from 0x%x ==> 0x%x\n",\
88 phba->attr_##_name, val); \
89 phba->attr_##_name = val;\
92 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
93 "BA_%d beiscsi_"#_name" attribute "\
94 "cannot be updated to 0x%x, "\
95 "range allowed is ["#_minval" - "#_maxval"]\n", val);\
99 #define beiscsi_store_param(_name) \
101 beiscsi_##_name##_store(struct device *dev,\
102 struct device_attribute *attr, const char *buf,\
105 struct Scsi_Host *shost = class_to_shost(dev);\
106 struct beiscsi_hba *phba = iscsi_host_priv(shost);\
107 uint32_t param_val = 0;\
108 if (!isdigit(buf[0]))\
110 if (sscanf(buf, "%i", ¶m_val) != 1)\
112 if (beiscsi_##_name##_change(phba, param_val) == 0) \
118 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \
120 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
122 if (val >= _minval && val <= _maxval) {\
123 phba->attr_##_name = val;\
126 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
127 "BA_%d beiscsi_"#_name" attribute " \
128 "cannot be updated to 0x%x, "\
129 "range allowed is ["#_minval" - "#_maxval"]\n", val);\
130 phba->attr_##_name = _defval;\
134 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
135 static uint beiscsi_##_name = _defval;\
136 module_param(beiscsi_##_name, uint, S_IRUGO);\
137 MODULE_PARM_DESC(beiscsi_##_name, _descp);\
138 beiscsi_disp_param(_name)\
139 beiscsi_change_param(_name, _minval, _maxval, _defval)\
140 beiscsi_store_param(_name)\
141 beiscsi_init_param(_name, _minval, _maxval, _defval)\
142 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
143 beiscsi_##_name##_disp, beiscsi_##_name##_store)
146 * When new log level added update the
147 * the MAX allowed value for log_enable
149 BEISCSI_RW_ATTR(log_enable
, 0x00,
150 0xFF, 0x00, "Enable logging Bit Mask\n"
151 "\t\t\t\tInitialization Events : 0x01\n"
152 "\t\t\t\tMailbox Events : 0x02\n"
153 "\t\t\t\tMiscellaneous Events : 0x04\n"
154 "\t\t\t\tError Handling : 0x08\n"
155 "\t\t\t\tIO Path Events : 0x10\n"
156 "\t\t\t\tConfiguration Path : 0x20\n"
157 "\t\t\t\tiSCSI Protocol : 0x40\n");
159 DEVICE_ATTR(beiscsi_drvr_ver
, S_IRUGO
, beiscsi_drvr_ver_disp
, NULL
);
160 DEVICE_ATTR(beiscsi_adapter_family
, S_IRUGO
, beiscsi_adap_family_disp
, NULL
);
161 DEVICE_ATTR(beiscsi_fw_ver
, S_IRUGO
, beiscsi_fw_ver_disp
, NULL
);
162 DEVICE_ATTR(beiscsi_phys_port
, S_IRUGO
, beiscsi_phys_port_disp
, NULL
);
163 DEVICE_ATTR(beiscsi_active_session_count
, S_IRUGO
,
164 beiscsi_active_session_disp
, NULL
);
165 DEVICE_ATTR(beiscsi_free_session_count
, S_IRUGO
,
166 beiscsi_free_session_disp
, NULL
);
167 struct device_attribute
*beiscsi_attrs
[] = {
168 &dev_attr_beiscsi_log_enable
,
169 &dev_attr_beiscsi_drvr_ver
,
170 &dev_attr_beiscsi_adapter_family
,
171 &dev_attr_beiscsi_fw_ver
,
172 &dev_attr_beiscsi_active_session_count
,
173 &dev_attr_beiscsi_free_session_count
,
174 &dev_attr_beiscsi_phys_port
,
178 static char const *cqe_desc
[] = {
181 "SOL_CMD_KILLED_DATA_DIGEST_ERR",
182 "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
183 "CXN_KILLED_BURST_LEN_MISMATCH",
184 "CXN_KILLED_AHS_RCVD",
185 "CXN_KILLED_HDR_DIGEST_ERR",
186 "CXN_KILLED_UNKNOWN_HDR",
187 "CXN_KILLED_STALE_ITT_TTT_RCVD",
188 "CXN_KILLED_INVALID_ITT_TTT_RCVD",
189 "CXN_KILLED_RST_RCVD",
190 "CXN_KILLED_TIMED_OUT",
191 "CXN_KILLED_RST_SENT",
192 "CXN_KILLED_FIN_RCVD",
193 "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
194 "CXN_KILLED_BAD_WRB_INDEX_ERROR",
195 "CXN_KILLED_OVER_RUN_RESIDUAL",
196 "CXN_KILLED_UNDER_RUN_RESIDUAL",
197 "CMD_KILLED_INVALID_STATSN_RCVD",
198 "CMD_KILLED_INVALID_R2T_RCVD",
199 "CMD_CXN_KILLED_LUN_INVALID",
200 "CMD_CXN_KILLED_ICD_INVALID",
201 "CMD_CXN_KILLED_ITT_INVALID",
202 "CMD_CXN_KILLED_SEQ_OUTOFORDER",
203 "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
204 "CXN_INVALIDATE_NOTIFY",
205 "CXN_INVALIDATE_INDEX_NOTIFY",
206 "CMD_INVALIDATED_NOTIFY",
209 "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
211 "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
212 "SOL_CMD_KILLED_DIF_ERR",
213 "CXN_KILLED_SYN_RCVD",
214 "CXN_KILLED_IMM_DATA_RCVD"
217 static int beiscsi_eh_abort(struct scsi_cmnd
*sc
)
219 struct iscsi_task
*abrt_task
= (struct iscsi_task
*)sc
->SCp
.ptr
;
220 struct iscsi_cls_session
*cls_session
;
221 struct beiscsi_io_task
*abrt_io_task
;
222 struct beiscsi_conn
*beiscsi_conn
;
223 struct iscsi_session
*session
;
224 struct invldt_cmd_tbl inv_tbl
;
225 struct beiscsi_hba
*phba
;
226 struct iscsi_conn
*conn
;
229 cls_session
= starget_to_session(scsi_target(sc
->device
));
230 session
= cls_session
->dd_data
;
232 /* check if we raced, task just got cleaned up under us */
233 spin_lock_bh(&session
->back_lock
);
234 if (!abrt_task
|| !abrt_task
->sc
) {
235 spin_unlock_bh(&session
->back_lock
);
238 /* get a task ref till FW processes the req for the ICD used */
239 __iscsi_get_task(abrt_task
);
240 abrt_io_task
= abrt_task
->dd_data
;
241 conn
= abrt_task
->conn
;
242 beiscsi_conn
= conn
->dd_data
;
243 phba
= beiscsi_conn
->phba
;
244 /* mark WRB invalid which have been not processed by FW yet */
245 if (is_chip_be2_be3r(phba
)) {
246 AMAP_SET_BITS(struct amap_iscsi_wrb
, invld
,
247 abrt_io_task
->pwrb_handle
->pwrb
, 1);
249 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, invld
,
250 abrt_io_task
->pwrb_handle
->pwrb
, 1);
252 inv_tbl
.cid
= beiscsi_conn
->beiscsi_conn_cid
;
253 inv_tbl
.icd
= abrt_io_task
->psgl_handle
->sgl_index
;
254 spin_unlock_bh(&session
->back_lock
);
256 rc
= beiscsi_mgmt_invalidate_icds(phba
, &inv_tbl
, 1);
257 iscsi_put_task(abrt_task
);
259 beiscsi_log(phba
, KERN_WARNING
, BEISCSI_LOG_EH
,
260 "BM_%d : sc %p invalidation failed %d\n",
265 return iscsi_eh_abort(sc
);
268 static int beiscsi_eh_device_reset(struct scsi_cmnd
*sc
)
270 struct beiscsi_invldt_cmd_tbl
{
271 struct invldt_cmd_tbl tbl
[BE_INVLDT_CMD_TBL_SZ
];
272 struct iscsi_task
*task
[BE_INVLDT_CMD_TBL_SZ
];
274 struct iscsi_cls_session
*cls_session
;
275 struct beiscsi_conn
*beiscsi_conn
;
276 struct beiscsi_io_task
*io_task
;
277 struct iscsi_session
*session
;
278 struct beiscsi_hba
*phba
;
279 struct iscsi_conn
*conn
;
280 struct iscsi_task
*task
;
281 unsigned int i
, nents
;
284 cls_session
= starget_to_session(scsi_target(sc
->device
));
285 session
= cls_session
->dd_data
;
287 spin_lock_bh(&session
->frwd_lock
);
288 if (!session
->leadconn
|| session
->state
!= ISCSI_STATE_LOGGED_IN
) {
289 spin_unlock_bh(&session
->frwd_lock
);
293 conn
= session
->leadconn
;
294 beiscsi_conn
= conn
->dd_data
;
295 phba
= beiscsi_conn
->phba
;
297 inv_tbl
= kzalloc(sizeof(*inv_tbl
), GFP_ATOMIC
);
299 spin_unlock_bh(&session
->frwd_lock
);
300 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_EH
,
301 "BM_%d : invldt_cmd_tbl alloc failed\n");
305 /* take back_lock to prevent task from getting cleaned up under us */
306 spin_lock(&session
->back_lock
);
307 for (i
= 0; i
< conn
->session
->cmds_max
; i
++) {
308 task
= conn
->session
->cmds
[i
];
312 if (sc
->device
->lun
!= task
->sc
->device
->lun
)
315 * Can't fit in more cmds? Normally this won't happen b'coz
316 * BEISCSI_CMD_PER_LUN is same as BE_INVLDT_CMD_TBL_SZ.
318 if (nents
== BE_INVLDT_CMD_TBL_SZ
) {
323 /* get a task ref till FW processes the req for the ICD used */
324 __iscsi_get_task(task
);
325 io_task
= task
->dd_data
;
326 /* mark WRB invalid which have been not processed by FW yet */
327 if (is_chip_be2_be3r(phba
)) {
328 AMAP_SET_BITS(struct amap_iscsi_wrb
, invld
,
329 io_task
->pwrb_handle
->pwrb
, 1);
331 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, invld
,
332 io_task
->pwrb_handle
->pwrb
, 1);
335 inv_tbl
->tbl
[nents
].cid
= beiscsi_conn
->beiscsi_conn_cid
;
336 inv_tbl
->tbl
[nents
].icd
= io_task
->psgl_handle
->sgl_index
;
337 inv_tbl
->task
[nents
] = task
;
340 spin_unlock(&session
->back_lock
);
341 spin_unlock_bh(&session
->frwd_lock
);
348 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_EH
,
349 "BM_%d : number of cmds exceeds size of invalidation table\n");
354 if (beiscsi_mgmt_invalidate_icds(phba
, &inv_tbl
->tbl
[0], nents
)) {
355 beiscsi_log(phba
, KERN_WARNING
, BEISCSI_LOG_EH
,
356 "BM_%d : cid %u scmds invalidation failed\n",
357 beiscsi_conn
->beiscsi_conn_cid
);
362 for (i
= 0; i
< nents
; i
++)
363 iscsi_put_task(inv_tbl
->task
[i
]);
367 rc
= iscsi_eh_device_reset(sc
);
371 /*------------------- PCI Driver operations and data ----------------- */
372 static const struct pci_device_id beiscsi_pci_id_table
[] = {
373 { PCI_DEVICE(BE_VENDOR_ID
, BE_DEVICE_ID1
) },
374 { PCI_DEVICE(BE_VENDOR_ID
, BE_DEVICE_ID2
) },
375 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID1
) },
376 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID2
) },
377 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID3
) },
378 { PCI_DEVICE(ELX_VENDOR_ID
, OC_SKH_ID1
) },
381 MODULE_DEVICE_TABLE(pci
, beiscsi_pci_id_table
);
384 static struct scsi_host_template beiscsi_sht
= {
385 .module
= THIS_MODULE
,
386 .name
= "Emulex 10Gbe open-iscsi Initiator Driver",
387 .proc_name
= DRV_NAME
,
388 .queuecommand
= iscsi_queuecommand
,
389 .change_queue_depth
= scsi_change_queue_depth
,
390 .target_alloc
= iscsi_target_alloc
,
391 .eh_timed_out
= iscsi_eh_cmd_timed_out
,
392 .eh_abort_handler
= beiscsi_eh_abort
,
393 .eh_device_reset_handler
= beiscsi_eh_device_reset
,
394 .eh_target_reset_handler
= iscsi_eh_session_reset
,
395 .shost_attrs
= beiscsi_attrs
,
396 .sg_tablesize
= BEISCSI_SGLIST_ELEMENTS
,
397 .can_queue
= BE2_IO_DEPTH
,
399 .max_sectors
= BEISCSI_MAX_SECTORS
,
400 .max_segment_size
= 65536,
401 .cmd_per_lun
= BEISCSI_CMD_PER_LUN
,
402 .vendor_id
= SCSI_NL_VID_TYPE_PCI
| BE_VENDOR_ID
,
403 .track_queue_depth
= 1,
406 static struct scsi_transport_template
*beiscsi_scsi_transport
;
408 static struct beiscsi_hba
*beiscsi_hba_alloc(struct pci_dev
*pcidev
)
410 struct beiscsi_hba
*phba
;
411 struct Scsi_Host
*shost
;
413 shost
= iscsi_host_alloc(&beiscsi_sht
, sizeof(*phba
), 0);
415 dev_err(&pcidev
->dev
,
416 "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
419 shost
->max_id
= BE2_MAX_SESSIONS
;
420 shost
->max_channel
= 0;
421 shost
->max_cmd_len
= BEISCSI_MAX_CMD_LEN
;
422 shost
->max_lun
= BEISCSI_NUM_MAX_LUN
;
423 shost
->transportt
= beiscsi_scsi_transport
;
424 phba
= iscsi_host_priv(shost
);
425 memset(phba
, 0, sizeof(*phba
));
427 phba
->pcidev
= pci_dev_get(pcidev
);
428 pci_set_drvdata(pcidev
, phba
);
429 phba
->interface_handle
= 0xFFFFFFFF;
434 static void beiscsi_unmap_pci_function(struct beiscsi_hba
*phba
)
437 iounmap(phba
->csr_va
);
441 iounmap(phba
->db_va
);
445 iounmap(phba
->pci_va
);
450 static int beiscsi_map_pci_bars(struct beiscsi_hba
*phba
,
451 struct pci_dev
*pcidev
)
456 addr
= ioremap_nocache(pci_resource_start(pcidev
, 2),
457 pci_resource_len(pcidev
, 2));
460 phba
->ctrl
.csr
= addr
;
463 addr
= ioremap_nocache(pci_resource_start(pcidev
, 4), 128 * 1024);
466 phba
->ctrl
.db
= addr
;
469 if (phba
->generation
== BE_GEN2
)
474 addr
= ioremap_nocache(pci_resource_start(pcidev
, pcicfg_reg
),
475 pci_resource_len(pcidev
, pcicfg_reg
));
479 phba
->ctrl
.pcicfg
= addr
;
484 beiscsi_unmap_pci_function(phba
);
488 static int beiscsi_enable_pci(struct pci_dev
*pcidev
)
492 ret
= pci_enable_device(pcidev
);
494 dev_err(&pcidev
->dev
,
495 "beiscsi_enable_pci - enable device failed\n");
499 ret
= pci_request_regions(pcidev
, DRV_NAME
);
501 dev_err(&pcidev
->dev
,
502 "beiscsi_enable_pci - request region failed\n");
503 goto pci_dev_disable
;
506 pci_set_master(pcidev
);
507 ret
= dma_set_mask_and_coherent(&pcidev
->dev
, DMA_BIT_MASK(64));
509 ret
= dma_set_mask_and_coherent(&pcidev
->dev
, DMA_BIT_MASK(32));
511 dev_err(&pcidev
->dev
, "Could not set PCI DMA Mask\n");
512 goto pci_region_release
;
518 pci_release_regions(pcidev
);
520 pci_disable_device(pcidev
);
525 static int be_ctrl_init(struct beiscsi_hba
*phba
, struct pci_dev
*pdev
)
527 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
528 struct be_dma_mem
*mbox_mem_alloc
= &ctrl
->mbox_mem_alloced
;
529 struct be_dma_mem
*mbox_mem_align
= &ctrl
->mbox_mem
;
533 status
= beiscsi_map_pci_bars(phba
, pdev
);
536 mbox_mem_alloc
->size
= sizeof(struct be_mcc_mailbox
) + 16;
537 mbox_mem_alloc
->va
= dma_alloc_coherent(&pdev
->dev
,
538 mbox_mem_alloc
->size
, &mbox_mem_alloc
->dma
, GFP_KERNEL
);
539 if (!mbox_mem_alloc
->va
) {
540 beiscsi_unmap_pci_function(phba
);
544 mbox_mem_align
->size
= sizeof(struct be_mcc_mailbox
);
545 mbox_mem_align
->va
= PTR_ALIGN(mbox_mem_alloc
->va
, 16);
546 mbox_mem_align
->dma
= PTR_ALIGN(mbox_mem_alloc
->dma
, 16);
547 memset(mbox_mem_align
->va
, 0, sizeof(struct be_mcc_mailbox
));
548 mutex_init(&ctrl
->mbox_lock
);
549 spin_lock_init(&phba
->ctrl
.mcc_lock
);
555 * beiscsi_get_params()- Set the config paramters
556 * @phba: ptr device priv structure
558 static void beiscsi_get_params(struct beiscsi_hba
*phba
)
560 uint32_t total_cid_count
= 0;
561 uint32_t total_icd_count
= 0;
564 total_cid_count
= BEISCSI_GET_CID_COUNT(phba
, BEISCSI_ULP0
) +
565 BEISCSI_GET_CID_COUNT(phba
, BEISCSI_ULP1
);
567 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
568 uint32_t align_mask
= 0;
569 uint32_t icd_post_per_page
= 0;
570 uint32_t icd_count_unavailable
= 0;
571 uint32_t icd_start
= 0, icd_count
= 0;
572 uint32_t icd_start_align
= 0, icd_count_align
= 0;
574 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
)) {
575 icd_start
= phba
->fw_config
.iscsi_icd_start
[ulp_num
];
576 icd_count
= phba
->fw_config
.iscsi_icd_count
[ulp_num
];
578 /* Get ICD count that can be posted on each page */
579 icd_post_per_page
= (PAGE_SIZE
/ (BE2_SGE
*
580 sizeof(struct iscsi_sge
)));
581 align_mask
= (icd_post_per_page
- 1);
583 /* Check if icd_start is aligned ICD per page posting */
584 if (icd_start
% icd_post_per_page
) {
585 icd_start_align
= ((icd_start
+
589 iscsi_icd_start
[ulp_num
] =
593 icd_count_align
= (icd_count
& ~align_mask
);
595 /* ICD discarded in the process of alignment */
597 icd_count_unavailable
= ((icd_start_align
-
602 /* Updated ICD count available */
603 phba
->fw_config
.iscsi_icd_count
[ulp_num
] = (icd_count
-
604 icd_count_unavailable
);
606 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
607 "BM_%d : Aligned ICD values\n"
608 "\t ICD Start : %d\n"
609 "\t ICD Count : %d\n"
610 "\t ICD Discarded : %d\n",
612 iscsi_icd_start
[ulp_num
],
614 iscsi_icd_count
[ulp_num
],
615 icd_count_unavailable
);
620 total_icd_count
= phba
->fw_config
.iscsi_icd_count
[ulp_num
];
621 phba
->params
.ios_per_ctrl
= (total_icd_count
-
623 BE2_TMFS
+ BE2_NOPOUT_REQ
));
624 phba
->params
.cxns_per_ctrl
= total_cid_count
;
625 phba
->params
.icds_per_ctrl
= total_icd_count
;
626 phba
->params
.num_sge_per_io
= BE2_SGE
;
627 phba
->params
.defpdu_hdr_sz
= BE2_DEFPDU_HDR_SZ
;
628 phba
->params
.defpdu_data_sz
= BE2_DEFPDU_DATA_SZ
;
629 phba
->params
.num_eq_entries
= 1024;
630 phba
->params
.num_cq_entries
= 1024;
631 phba
->params
.wrbs_per_cxn
= 256;
634 static void hwi_ring_eq_db(struct beiscsi_hba
*phba
,
635 unsigned int id
, unsigned int clr_interrupt
,
636 unsigned int num_processed
,
637 unsigned char rearm
, unsigned char event
)
642 val
|= 1 << DB_EQ_REARM_SHIFT
;
644 val
|= 1 << DB_EQ_CLR_SHIFT
;
646 val
|= 1 << DB_EQ_EVNT_SHIFT
;
648 val
|= num_processed
<< DB_EQ_NUM_POPPED_SHIFT
;
649 /* Setting lower order EQ_ID Bits */
650 val
|= (id
& DB_EQ_RING_ID_LOW_MASK
);
652 /* Setting Higher order EQ_ID Bits */
653 val
|= (((id
>> DB_EQ_HIGH_FEILD_SHIFT
) &
654 DB_EQ_RING_ID_HIGH_MASK
)
655 << DB_EQ_HIGH_SET_SHIFT
);
657 iowrite32(val
, phba
->db_va
+ DB_EQ_OFFSET
);
661 * be_isr_mcc - The isr routine of the driver.
663 * @dev_id: Pointer to host adapter structure
665 static irqreturn_t
be_isr_mcc(int irq
, void *dev_id
)
667 struct beiscsi_hba
*phba
;
668 struct be_eq_entry
*eqe
;
669 struct be_queue_info
*eq
;
670 struct be_queue_info
*mcc
;
671 unsigned int mcc_events
;
672 struct be_eq_obj
*pbe_eq
;
677 mcc
= &phba
->ctrl
.mcc_obj
.cq
;
678 eqe
= queue_tail_node(eq
);
681 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
683 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
685 EQE_RESID_MASK
) >> 16) == mcc
->id
) {
688 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
690 eqe
= queue_tail_node(eq
);
694 queue_work(phba
->wq
, &pbe_eq
->mcc_work
);
695 hwi_ring_eq_db(phba
, eq
->id
, 1, mcc_events
, 1, 1);
701 * be_isr_msix - The isr routine of the driver.
703 * @dev_id: Pointer to host adapter structure
705 static irqreturn_t
be_isr_msix(int irq
, void *dev_id
)
707 struct beiscsi_hba
*phba
;
708 struct be_queue_info
*eq
;
709 struct be_eq_obj
*pbe_eq
;
715 /* disable interrupt till iopoll completes */
716 hwi_ring_eq_db(phba
, eq
->id
, 1, 0, 0, 1);
717 irq_poll_sched(&pbe_eq
->iopoll
);
723 * be_isr - The isr routine of the driver.
725 * @dev_id: Pointer to host adapter structure
727 static irqreturn_t
be_isr(int irq
, void *dev_id
)
729 struct beiscsi_hba
*phba
;
730 struct hwi_controller
*phwi_ctrlr
;
731 struct hwi_context_memory
*phwi_context
;
732 struct be_eq_entry
*eqe
;
733 struct be_queue_info
*eq
;
734 struct be_queue_info
*mcc
;
735 unsigned int mcc_events
, io_events
;
736 struct be_ctrl_info
*ctrl
;
737 struct be_eq_obj
*pbe_eq
;
742 isr
= ioread32(ctrl
->csr
+ CEV_ISR0_OFFSET
+
743 (PCI_FUNC(ctrl
->pdev
->devfn
) * CEV_ISR_SIZE
));
747 phwi_ctrlr
= phba
->phwi_ctrlr
;
748 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
749 pbe_eq
= &phwi_context
->be_eq
[0];
751 eq
= &phwi_context
->be_eq
[0].q
;
752 mcc
= &phba
->ctrl
.mcc_obj
.cq
;
753 eqe
= queue_tail_node(eq
);
757 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
759 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
760 resource_id
) / 32] & EQE_RESID_MASK
) >> 16) == mcc
->id
)
764 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
766 eqe
= queue_tail_node(eq
);
768 if (!io_events
&& !mcc_events
)
771 /* no need to rearm if interrupt is only for IOs */
774 queue_work(phba
->wq
, &pbe_eq
->mcc_work
);
779 irq_poll_sched(&pbe_eq
->iopoll
);
780 hwi_ring_eq_db(phba
, eq
->id
, 0, (io_events
+ mcc_events
), rearm
, 1);
784 static void beiscsi_free_irqs(struct beiscsi_hba
*phba
)
786 struct hwi_context_memory
*phwi_context
;
789 if (!phba
->pcidev
->msix_enabled
) {
790 if (phba
->pcidev
->irq
)
791 free_irq(phba
->pcidev
->irq
, phba
);
795 phwi_context
= phba
->phwi_ctrlr
->phwi_ctxt
;
796 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
797 free_irq(pci_irq_vector(phba
->pcidev
, i
),
798 &phwi_context
->be_eq
[i
]);
799 kfree(phba
->msi_name
[i
]);
803 static int beiscsi_init_irqs(struct beiscsi_hba
*phba
)
805 struct pci_dev
*pcidev
= phba
->pcidev
;
806 struct hwi_controller
*phwi_ctrlr
;
807 struct hwi_context_memory
*phwi_context
;
810 phwi_ctrlr
= phba
->phwi_ctrlr
;
811 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
813 if (pcidev
->msix_enabled
) {
814 for (i
= 0; i
< phba
->num_cpus
; i
++) {
815 phba
->msi_name
[i
] = kasprintf(GFP_KERNEL
,
817 phba
->shost
->host_no
, i
);
818 if (!phba
->msi_name
[i
]) {
823 ret
= request_irq(pci_irq_vector(pcidev
, i
),
824 be_isr_msix
, 0, phba
->msi_name
[i
],
825 &phwi_context
->be_eq
[i
]);
827 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
828 "BM_%d : beiscsi_init_irqs-Failed to"
829 "register msix for i = %d\n",
831 kfree(phba
->msi_name
[i
]);
835 phba
->msi_name
[i
] = kasprintf(GFP_KERNEL
, "beiscsi_mcc_%02x",
836 phba
->shost
->host_no
);
837 if (!phba
->msi_name
[i
]) {
841 ret
= request_irq(pci_irq_vector(pcidev
, i
), be_isr_mcc
, 0,
842 phba
->msi_name
[i
], &phwi_context
->be_eq
[i
]);
844 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
845 "BM_%d : beiscsi_init_irqs-"
846 "Failed to register beiscsi_msix_mcc\n");
847 kfree(phba
->msi_name
[i
]);
852 ret
= request_irq(pcidev
->irq
, be_isr
, IRQF_SHARED
,
855 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
856 "BM_%d : beiscsi_init_irqs-"
857 "Failed to register irq\\n");
863 for (j
= i
- 1; j
>= 0; j
--) {
864 free_irq(pci_irq_vector(pcidev
, i
), &phwi_context
->be_eq
[j
]);
865 kfree(phba
->msi_name
[j
]);
870 void hwi_ring_cq_db(struct beiscsi_hba
*phba
,
871 unsigned int id
, unsigned int num_processed
,
877 val
|= 1 << DB_CQ_REARM_SHIFT
;
879 val
|= num_processed
<< DB_CQ_NUM_POPPED_SHIFT
;
881 /* Setting lower order CQ_ID Bits */
882 val
|= (id
& DB_CQ_RING_ID_LOW_MASK
);
884 /* Setting Higher order CQ_ID Bits */
885 val
|= (((id
>> DB_CQ_HIGH_FEILD_SHIFT
) &
886 DB_CQ_RING_ID_HIGH_MASK
)
887 << DB_CQ_HIGH_SET_SHIFT
);
889 iowrite32(val
, phba
->db_va
+ DB_CQ_OFFSET
);
892 static struct sgl_handle
*alloc_io_sgl_handle(struct beiscsi_hba
*phba
)
894 struct sgl_handle
*psgl_handle
;
897 spin_lock_irqsave(&phba
->io_sgl_lock
, flags
);
898 if (phba
->io_sgl_hndl_avbl
) {
899 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_IO
,
900 "BM_%d : In alloc_io_sgl_handle,"
901 " io_sgl_alloc_index=%d\n",
902 phba
->io_sgl_alloc_index
);
904 psgl_handle
= phba
->io_sgl_hndl_base
[phba
->
906 phba
->io_sgl_hndl_base
[phba
->io_sgl_alloc_index
] = NULL
;
907 phba
->io_sgl_hndl_avbl
--;
908 if (phba
->io_sgl_alloc_index
== (phba
->params
.
910 phba
->io_sgl_alloc_index
= 0;
912 phba
->io_sgl_alloc_index
++;
915 spin_unlock_irqrestore(&phba
->io_sgl_lock
, flags
);
920 free_io_sgl_handle(struct beiscsi_hba
*phba
, struct sgl_handle
*psgl_handle
)
924 spin_lock_irqsave(&phba
->io_sgl_lock
, flags
);
925 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_IO
,
926 "BM_%d : In free_,io_sgl_free_index=%d\n",
927 phba
->io_sgl_free_index
);
929 if (phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
]) {
931 * this can happen if clean_task is called on a task that
932 * failed in xmit_task or alloc_pdu.
934 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_IO
,
935 "BM_%d : Double Free in IO SGL io_sgl_free_index=%d, value there=%p\n",
936 phba
->io_sgl_free_index
,
937 phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
]);
938 spin_unlock_irqrestore(&phba
->io_sgl_lock
, flags
);
941 phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
] = psgl_handle
;
942 phba
->io_sgl_hndl_avbl
++;
943 if (phba
->io_sgl_free_index
== (phba
->params
.ios_per_ctrl
- 1))
944 phba
->io_sgl_free_index
= 0;
946 phba
->io_sgl_free_index
++;
947 spin_unlock_irqrestore(&phba
->io_sgl_lock
, flags
);
950 static inline struct wrb_handle
*
951 beiscsi_get_wrb_handle(struct hwi_wrb_context
*pwrb_context
,
952 unsigned int wrbs_per_cxn
)
954 struct wrb_handle
*pwrb_handle
;
957 spin_lock_irqsave(&pwrb_context
->wrb_lock
, flags
);
958 if (!pwrb_context
->wrb_handles_available
) {
959 spin_unlock_irqrestore(&pwrb_context
->wrb_lock
, flags
);
962 pwrb_handle
= pwrb_context
->pwrb_handle_base
[pwrb_context
->alloc_index
];
963 pwrb_context
->wrb_handles_available
--;
964 if (pwrb_context
->alloc_index
== (wrbs_per_cxn
- 1))
965 pwrb_context
->alloc_index
= 0;
967 pwrb_context
->alloc_index
++;
968 spin_unlock_irqrestore(&pwrb_context
->wrb_lock
, flags
);
971 memset(pwrb_handle
->pwrb
, 0, sizeof(*pwrb_handle
->pwrb
));
977 * alloc_wrb_handle - To allocate a wrb handle
978 * @phba: The hba pointer
979 * @cid: The cid to use for allocation
980 * @pwrb_context: ptr to ptr to wrb context
982 * This happens under session_lock until submission to chip
984 struct wrb_handle
*alloc_wrb_handle(struct beiscsi_hba
*phba
, unsigned int cid
,
985 struct hwi_wrb_context
**pcontext
)
987 struct hwi_wrb_context
*pwrb_context
;
988 struct hwi_controller
*phwi_ctrlr
;
989 uint16_t cri_index
= BE_GET_CRI_FROM_CID(cid
);
991 phwi_ctrlr
= phba
->phwi_ctrlr
;
992 pwrb_context
= &phwi_ctrlr
->wrb_context
[cri_index
];
993 /* return the context address */
994 *pcontext
= pwrb_context
;
995 return beiscsi_get_wrb_handle(pwrb_context
, phba
->params
.wrbs_per_cxn
);
999 beiscsi_put_wrb_handle(struct hwi_wrb_context
*pwrb_context
,
1000 struct wrb_handle
*pwrb_handle
,
1001 unsigned int wrbs_per_cxn
)
1003 unsigned long flags
;
1005 spin_lock_irqsave(&pwrb_context
->wrb_lock
, flags
);
1006 pwrb_context
->pwrb_handle_base
[pwrb_context
->free_index
] = pwrb_handle
;
1007 pwrb_context
->wrb_handles_available
++;
1008 if (pwrb_context
->free_index
== (wrbs_per_cxn
- 1))
1009 pwrb_context
->free_index
= 0;
1011 pwrb_context
->free_index
++;
1012 pwrb_handle
->pio_handle
= NULL
;
1013 spin_unlock_irqrestore(&pwrb_context
->wrb_lock
, flags
);
1017 * free_wrb_handle - To free the wrb handle back to pool
1018 * @phba: The hba pointer
1019 * @pwrb_context: The context to free from
1020 * @pwrb_handle: The wrb_handle to free
1022 * This happens under session_lock until submission to chip
1025 free_wrb_handle(struct beiscsi_hba
*phba
, struct hwi_wrb_context
*pwrb_context
,
1026 struct wrb_handle
*pwrb_handle
)
1028 beiscsi_put_wrb_handle(pwrb_context
,
1030 phba
->params
.wrbs_per_cxn
);
1031 beiscsi_log(phba
, KERN_INFO
,
1032 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
1033 "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1034 "wrb_handles_available=%d\n",
1035 pwrb_handle
, pwrb_context
->free_index
,
1036 pwrb_context
->wrb_handles_available
);
1039 static struct sgl_handle
*alloc_mgmt_sgl_handle(struct beiscsi_hba
*phba
)
1041 struct sgl_handle
*psgl_handle
;
1042 unsigned long flags
;
1044 spin_lock_irqsave(&phba
->mgmt_sgl_lock
, flags
);
1045 if (phba
->eh_sgl_hndl_avbl
) {
1046 psgl_handle
= phba
->eh_sgl_hndl_base
[phba
->eh_sgl_alloc_index
];
1047 phba
->eh_sgl_hndl_base
[phba
->eh_sgl_alloc_index
] = NULL
;
1048 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_CONFIG
,
1049 "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1050 phba
->eh_sgl_alloc_index
,
1051 phba
->eh_sgl_alloc_index
);
1053 phba
->eh_sgl_hndl_avbl
--;
1054 if (phba
->eh_sgl_alloc_index
==
1055 (phba
->params
.icds_per_ctrl
- phba
->params
.ios_per_ctrl
-
1057 phba
->eh_sgl_alloc_index
= 0;
1059 phba
->eh_sgl_alloc_index
++;
1062 spin_unlock_irqrestore(&phba
->mgmt_sgl_lock
, flags
);
1067 free_mgmt_sgl_handle(struct beiscsi_hba
*phba
, struct sgl_handle
*psgl_handle
)
1069 unsigned long flags
;
1071 spin_lock_irqsave(&phba
->mgmt_sgl_lock
, flags
);
1072 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_CONFIG
,
1073 "BM_%d : In free_mgmt_sgl_handle,"
1074 "eh_sgl_free_index=%d\n",
1075 phba
->eh_sgl_free_index
);
1077 if (phba
->eh_sgl_hndl_base
[phba
->eh_sgl_free_index
]) {
1079 * this can happen if clean_task is called on a task that
1080 * failed in xmit_task or alloc_pdu.
1082 beiscsi_log(phba
, KERN_WARNING
, BEISCSI_LOG_CONFIG
,
1083 "BM_%d : Double Free in eh SGL ,"
1084 "eh_sgl_free_index=%d\n",
1085 phba
->eh_sgl_free_index
);
1086 spin_unlock_irqrestore(&phba
->mgmt_sgl_lock
, flags
);
1089 phba
->eh_sgl_hndl_base
[phba
->eh_sgl_free_index
] = psgl_handle
;
1090 phba
->eh_sgl_hndl_avbl
++;
1091 if (phba
->eh_sgl_free_index
==
1092 (phba
->params
.icds_per_ctrl
- phba
->params
.ios_per_ctrl
- 1))
1093 phba
->eh_sgl_free_index
= 0;
1095 phba
->eh_sgl_free_index
++;
1096 spin_unlock_irqrestore(&phba
->mgmt_sgl_lock
, flags
);
1100 be_complete_io(struct beiscsi_conn
*beiscsi_conn
,
1101 struct iscsi_task
*task
,
1102 struct common_sol_cqe
*csol_cqe
)
1104 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1105 struct be_status_bhs
*sts_bhs
=
1106 (struct be_status_bhs
*)io_task
->cmd_bhs
;
1107 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1108 unsigned char *sense
;
1109 u32 resid
= 0, exp_cmdsn
, max_cmdsn
;
1110 u8 rsp
, status
, flags
;
1112 exp_cmdsn
= csol_cqe
->exp_cmdsn
;
1113 max_cmdsn
= (csol_cqe
->exp_cmdsn
+
1114 csol_cqe
->cmd_wnd
- 1);
1115 rsp
= csol_cqe
->i_resp
;
1116 status
= csol_cqe
->i_sts
;
1117 flags
= csol_cqe
->i_flags
;
1118 resid
= csol_cqe
->res_cnt
;
1121 if (io_task
->scsi_cmnd
) {
1122 scsi_dma_unmap(io_task
->scsi_cmnd
);
1123 io_task
->scsi_cmnd
= NULL
;
1128 task
->sc
->result
= (DID_OK
<< 16) | status
;
1129 if (rsp
!= ISCSI_STATUS_CMD_COMPLETED
) {
1130 task
->sc
->result
= DID_ERROR
<< 16;
1134 /* bidi not initially supported */
1135 if (flags
& (ISCSI_FLAG_CMD_UNDERFLOW
| ISCSI_FLAG_CMD_OVERFLOW
)) {
1136 if (!status
&& (flags
& ISCSI_FLAG_CMD_OVERFLOW
))
1137 task
->sc
->result
= DID_ERROR
<< 16;
1139 if (flags
& ISCSI_FLAG_CMD_UNDERFLOW
) {
1140 scsi_set_resid(task
->sc
, resid
);
1141 if (!status
&& (scsi_bufflen(task
->sc
) - resid
<
1142 task
->sc
->underflow
))
1143 task
->sc
->result
= DID_ERROR
<< 16;
1147 if (status
== SAM_STAT_CHECK_CONDITION
) {
1149 unsigned short *slen
= (unsigned short *)sts_bhs
->sense_info
;
1151 sense
= sts_bhs
->sense_info
+ sizeof(unsigned short);
1152 sense_len
= be16_to_cpu(*slen
);
1153 memcpy(task
->sc
->sense_buffer
, sense
,
1154 min_t(u16
, sense_len
, SCSI_SENSE_BUFFERSIZE
));
1157 if (io_task
->cmd_bhs
->iscsi_hdr
.flags
& ISCSI_FLAG_CMD_READ
)
1158 conn
->rxdata_octets
+= resid
;
1160 if (io_task
->scsi_cmnd
) {
1161 scsi_dma_unmap(io_task
->scsi_cmnd
);
1162 io_task
->scsi_cmnd
= NULL
;
1164 iscsi_complete_scsi_task(task
, exp_cmdsn
, max_cmdsn
);
1168 be_complete_logout(struct beiscsi_conn
*beiscsi_conn
,
1169 struct iscsi_task
*task
,
1170 struct common_sol_cqe
*csol_cqe
)
1172 struct iscsi_logout_rsp
*hdr
;
1173 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1174 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1176 hdr
= (struct iscsi_logout_rsp
*)task
->hdr
;
1177 hdr
->opcode
= ISCSI_OP_LOGOUT_RSP
;
1180 hdr
->flags
= csol_cqe
->i_flags
;
1181 hdr
->response
= csol_cqe
->i_resp
;
1182 hdr
->exp_cmdsn
= cpu_to_be32(csol_cqe
->exp_cmdsn
);
1183 hdr
->max_cmdsn
= cpu_to_be32(csol_cqe
->exp_cmdsn
+
1184 csol_cqe
->cmd_wnd
- 1);
1186 hdr
->dlength
[0] = 0;
1187 hdr
->dlength
[1] = 0;
1188 hdr
->dlength
[2] = 0;
1190 hdr
->itt
= io_task
->libiscsi_itt
;
1191 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1195 be_complete_tmf(struct beiscsi_conn
*beiscsi_conn
,
1196 struct iscsi_task
*task
,
1197 struct common_sol_cqe
*csol_cqe
)
1199 struct iscsi_tm_rsp
*hdr
;
1200 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1201 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1203 hdr
= (struct iscsi_tm_rsp
*)task
->hdr
;
1204 hdr
->opcode
= ISCSI_OP_SCSI_TMFUNC_RSP
;
1205 hdr
->flags
= csol_cqe
->i_flags
;
1206 hdr
->response
= csol_cqe
->i_resp
;
1207 hdr
->exp_cmdsn
= cpu_to_be32(csol_cqe
->exp_cmdsn
);
1208 hdr
->max_cmdsn
= cpu_to_be32(csol_cqe
->exp_cmdsn
+
1209 csol_cqe
->cmd_wnd
- 1);
1211 hdr
->itt
= io_task
->libiscsi_itt
;
1212 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1216 hwi_complete_drvr_msgs(struct beiscsi_conn
*beiscsi_conn
,
1217 struct beiscsi_hba
*phba
, struct sol_cqe
*psol
)
1219 struct hwi_wrb_context
*pwrb_context
;
1220 uint16_t wrb_index
, cid
, cri_index
;
1221 struct hwi_controller
*phwi_ctrlr
;
1222 struct wrb_handle
*pwrb_handle
;
1223 struct iscsi_session
*session
;
1224 struct iscsi_task
*task
;
1226 phwi_ctrlr
= phba
->phwi_ctrlr
;
1227 if (is_chip_be2_be3r(phba
)) {
1228 wrb_index
= AMAP_GET_BITS(struct amap_it_dmsg_cqe
,
1230 cid
= AMAP_GET_BITS(struct amap_it_dmsg_cqe
,
1233 wrb_index
= AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2
,
1235 cid
= AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2
,
1239 cri_index
= BE_GET_CRI_FROM_CID(cid
);
1240 pwrb_context
= &phwi_ctrlr
->wrb_context
[cri_index
];
1241 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[wrb_index
];
1242 session
= beiscsi_conn
->conn
->session
;
1243 spin_lock_bh(&session
->back_lock
);
1244 task
= pwrb_handle
->pio_handle
;
1246 __iscsi_put_task(task
);
1247 spin_unlock_bh(&session
->back_lock
);
1251 be_complete_nopin_resp(struct beiscsi_conn
*beiscsi_conn
,
1252 struct iscsi_task
*task
,
1253 struct common_sol_cqe
*csol_cqe
)
1255 struct iscsi_nopin
*hdr
;
1256 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1257 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1259 hdr
= (struct iscsi_nopin
*)task
->hdr
;
1260 hdr
->flags
= csol_cqe
->i_flags
;
1261 hdr
->exp_cmdsn
= cpu_to_be32(csol_cqe
->exp_cmdsn
);
1262 hdr
->max_cmdsn
= cpu_to_be32(csol_cqe
->exp_cmdsn
+
1263 csol_cqe
->cmd_wnd
- 1);
1265 hdr
->opcode
= ISCSI_OP_NOOP_IN
;
1266 hdr
->itt
= io_task
->libiscsi_itt
;
1267 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1270 static void adapter_get_sol_cqe(struct beiscsi_hba
*phba
,
1271 struct sol_cqe
*psol
,
1272 struct common_sol_cqe
*csol_cqe
)
1274 if (is_chip_be2_be3r(phba
)) {
1275 csol_cqe
->exp_cmdsn
= AMAP_GET_BITS(struct amap_sol_cqe
,
1276 i_exp_cmd_sn
, psol
);
1277 csol_cqe
->res_cnt
= AMAP_GET_BITS(struct amap_sol_cqe
,
1279 csol_cqe
->cmd_wnd
= AMAP_GET_BITS(struct amap_sol_cqe
,
1281 csol_cqe
->wrb_index
= AMAP_GET_BITS(struct amap_sol_cqe
,
1283 csol_cqe
->cid
= AMAP_GET_BITS(struct amap_sol_cqe
,
1285 csol_cqe
->hw_sts
= AMAP_GET_BITS(struct amap_sol_cqe
,
1287 csol_cqe
->i_resp
= AMAP_GET_BITS(struct amap_sol_cqe
,
1289 csol_cqe
->i_sts
= AMAP_GET_BITS(struct amap_sol_cqe
,
1291 csol_cqe
->i_flags
= AMAP_GET_BITS(struct amap_sol_cqe
,
1294 csol_cqe
->exp_cmdsn
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1295 i_exp_cmd_sn
, psol
);
1296 csol_cqe
->res_cnt
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1298 csol_cqe
->wrb_index
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1300 csol_cqe
->cid
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1302 csol_cqe
->hw_sts
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1304 csol_cqe
->cmd_wnd
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1306 if (AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1308 csol_cqe
->i_sts
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1311 csol_cqe
->i_resp
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1313 if (AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1315 csol_cqe
->i_flags
= ISCSI_FLAG_CMD_UNDERFLOW
;
1317 if (AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1319 csol_cqe
->i_flags
|= ISCSI_FLAG_CMD_OVERFLOW
;
1324 static void hwi_complete_cmd(struct beiscsi_conn
*beiscsi_conn
,
1325 struct beiscsi_hba
*phba
, struct sol_cqe
*psol
)
1327 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1328 struct iscsi_session
*session
= conn
->session
;
1329 struct common_sol_cqe csol_cqe
= {0};
1330 struct hwi_wrb_context
*pwrb_context
;
1331 struct hwi_controller
*phwi_ctrlr
;
1332 struct wrb_handle
*pwrb_handle
;
1333 struct iscsi_task
*task
;
1334 uint16_t cri_index
= 0;
1337 phwi_ctrlr
= phba
->phwi_ctrlr
;
1339 /* Copy the elements to a common structure */
1340 adapter_get_sol_cqe(phba
, psol
, &csol_cqe
);
1342 cri_index
= BE_GET_CRI_FROM_CID(csol_cqe
.cid
);
1343 pwrb_context
= &phwi_ctrlr
->wrb_context
[cri_index
];
1345 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[
1346 csol_cqe
.wrb_index
];
1348 spin_lock_bh(&session
->back_lock
);
1349 task
= pwrb_handle
->pio_handle
;
1351 spin_unlock_bh(&session
->back_lock
);
1354 type
= ((struct beiscsi_io_task
*)task
->dd_data
)->wrb_type
;
1358 case HWH_TYPE_IO_RD
:
1359 if ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) ==
1361 be_complete_nopin_resp(beiscsi_conn
, task
, &csol_cqe
);
1363 be_complete_io(beiscsi_conn
, task
, &csol_cqe
);
1366 case HWH_TYPE_LOGOUT
:
1367 if ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGOUT
)
1368 be_complete_logout(beiscsi_conn
, task
, &csol_cqe
);
1370 be_complete_tmf(beiscsi_conn
, task
, &csol_cqe
);
1373 case HWH_TYPE_LOGIN
:
1374 beiscsi_log(phba
, KERN_ERR
,
1375 BEISCSI_LOG_CONFIG
| BEISCSI_LOG_IO
,
1376 "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1377 " hwi_complete_cmd- Solicited path\n");
1381 be_complete_nopin_resp(beiscsi_conn
, task
, &csol_cqe
);
1385 beiscsi_log(phba
, KERN_WARNING
,
1386 BEISCSI_LOG_CONFIG
| BEISCSI_LOG_IO
,
1387 "BM_%d : In hwi_complete_cmd, unknown type = %d"
1388 "wrb_index 0x%x CID 0x%x\n", type
,
1394 spin_unlock_bh(&session
->back_lock
);
1398 * ASYNC PDUs include
1399 * a. Unsolicited NOP-In (target initiated NOP-In)
1403 * These headers arrive unprocessed by the EP firmware.
1404 * iSCSI layer processes them.
1407 beiscsi_complete_pdu(struct beiscsi_conn
*beiscsi_conn
,
1408 struct pdu_base
*phdr
, void *pdata
, unsigned int dlen
)
1410 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
1411 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1412 struct beiscsi_io_task
*io_task
;
1413 struct iscsi_hdr
*login_hdr
;
1414 struct iscsi_task
*task
;
1417 code
= AMAP_GET_BITS(struct amap_pdu_base
, opcode
, phdr
);
1419 case ISCSI_OP_NOOP_IN
:
1423 case ISCSI_OP_ASYNC_EVENT
:
1425 case ISCSI_OP_REJECT
:
1427 WARN_ON(!(dlen
== 48));
1428 beiscsi_log(phba
, KERN_ERR
,
1429 BEISCSI_LOG_CONFIG
| BEISCSI_LOG_IO
,
1430 "BM_%d : In ISCSI_OP_REJECT\n");
1432 case ISCSI_OP_LOGIN_RSP
:
1433 case ISCSI_OP_TEXT_RSP
:
1434 task
= conn
->login_task
;
1435 io_task
= task
->dd_data
;
1436 login_hdr
= (struct iscsi_hdr
*)phdr
;
1437 login_hdr
->itt
= io_task
->libiscsi_itt
;
1440 beiscsi_log(phba
, KERN_WARNING
,
1441 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
1442 "BM_%d : unrecognized async PDU opcode 0x%x\n",
1446 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)phdr
, pdata
, dlen
);
1451 beiscsi_hdl_put_handle(struct hd_async_context
*pasync_ctx
,
1452 struct hd_async_handle
*pasync_handle
)
1454 pasync_handle
->is_final
= 0;
1455 pasync_handle
->buffer_len
= 0;
1456 pasync_handle
->in_use
= 0;
1457 list_del_init(&pasync_handle
->link
);
1461 beiscsi_hdl_purge_handles(struct beiscsi_hba
*phba
,
1462 struct hd_async_context
*pasync_ctx
,
1465 struct hd_async_handle
*pasync_handle
, *tmp_handle
;
1466 struct list_head
*plist
;
1468 plist
= &pasync_ctx
->async_entry
[cri
].wq
.list
;
1469 list_for_each_entry_safe(pasync_handle
, tmp_handle
, plist
, link
)
1470 beiscsi_hdl_put_handle(pasync_ctx
, pasync_handle
);
1472 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[cri
].wq
.list
);
1473 pasync_ctx
->async_entry
[cri
].wq
.hdr_len
= 0;
1474 pasync_ctx
->async_entry
[cri
].wq
.bytes_received
= 0;
1475 pasync_ctx
->async_entry
[cri
].wq
.bytes_needed
= 0;
1478 static struct hd_async_handle
*
1479 beiscsi_hdl_get_handle(struct beiscsi_conn
*beiscsi_conn
,
1480 struct hd_async_context
*pasync_ctx
,
1481 struct i_t_dpdu_cqe
*pdpdu_cqe
,
1484 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
1485 struct hd_async_handle
*pasync_handle
;
1486 struct be_bus_address phys_addr
;
1487 u16 cid
, code
, ci
, cri
;
1488 u8 final
, error
= 0;
1491 cid
= beiscsi_conn
->beiscsi_conn_cid
;
1492 cri
= BE_GET_ASYNC_CRI_FROM_CID(cid
);
1494 * This function is invoked to get the right async_handle structure
1495 * from a given DEF PDU CQ entry.
1497 * - index in CQ entry gives the vertical index
1498 * - address in CQ entry is the offset where the DMA last ended
1499 * - final - no more notifications for this PDU
1501 if (is_chip_be2_be3r(phba
)) {
1502 dpl
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe
,
1504 ci
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe
,
1506 final
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe
,
1509 dpl
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2
,
1511 ci
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2
,
1513 final
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2
,
1518 * DB addr Hi/Lo is same for BE and SKH.
1519 * Subtract the dataplacementlength to get to the base.
1521 phys_addr
.u
.a32
.address_lo
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe
,
1522 db_addr_lo
, pdpdu_cqe
);
1523 phys_addr
.u
.a32
.address_lo
-= dpl
;
1524 phys_addr
.u
.a32
.address_hi
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe
,
1525 db_addr_hi
, pdpdu_cqe
);
1527 code
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe
, code
, pdpdu_cqe
);
1529 case UNSOL_HDR_NOTIFY
:
1530 pasync_handle
= pasync_ctx
->async_entry
[ci
].header
;
1533 case UNSOL_DATA_DIGEST_ERROR_NOTIFY
:
1535 case UNSOL_DATA_NOTIFY
:
1536 pasync_handle
= pasync_ctx
->async_entry
[ci
].data
;
1538 /* called only for above codes */
1543 if (pasync_handle
->pa
.u
.a64
.address
!= phys_addr
.u
.a64
.address
||
1544 pasync_handle
->index
!= ci
) {
1545 /* driver bug - if ci does not match async handle index */
1547 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_ISCSI
,
1548 "BM_%d : cid %u async PDU handle mismatch - addr in %cQE %llx at %u:addr in CQE %llx ci %u\n",
1549 cid
, pasync_handle
->is_header
? 'H' : 'D',
1550 pasync_handle
->pa
.u
.a64
.address
,
1551 pasync_handle
->index
,
1552 phys_addr
.u
.a64
.address
, ci
);
1553 /* FW has stale address - attempt continuing by dropping */
1557 * DEF PDU header and data buffers with errors should be simply
1558 * dropped as there are no consumers for it.
1561 beiscsi_hdl_put_handle(pasync_ctx
, pasync_handle
);
1565 if (pasync_handle
->in_use
|| !list_empty(&pasync_handle
->link
)) {
1566 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_ISCSI
,
1567 "BM_%d : cid %d async PDU handle in use - code %d ci %d addr %llx\n",
1568 cid
, code
, ci
, phys_addr
.u
.a64
.address
);
1569 beiscsi_hdl_purge_handles(phba
, pasync_ctx
, cri
);
1572 list_del_init(&pasync_handle
->link
);
1574 * Each CID is associated with unique CRI.
1575 * ASYNC_CRI_FROM_CID mapping and CRI_FROM_CID are totaly different.
1577 pasync_handle
->cri
= cri
;
1578 pasync_handle
->is_final
= final
;
1579 pasync_handle
->buffer_len
= dpl
;
1580 pasync_handle
->in_use
= 1;
1582 return pasync_handle
;
1586 beiscsi_hdl_fwd_pdu(struct beiscsi_conn
*beiscsi_conn
,
1587 struct hd_async_context
*pasync_ctx
,
1590 struct iscsi_session
*session
= beiscsi_conn
->conn
->session
;
1591 struct hd_async_handle
*pasync_handle
, *plast_handle
;
1592 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
1593 void *phdr
= NULL
, *pdata
= NULL
;
1594 u32 dlen
= 0, status
= 0;
1595 struct list_head
*plist
;
1597 plist
= &pasync_ctx
->async_entry
[cri
].wq
.list
;
1598 plast_handle
= NULL
;
1599 list_for_each_entry(pasync_handle
, plist
, link
) {
1600 plast_handle
= pasync_handle
;
1601 /* get the header, the first entry */
1603 phdr
= pasync_handle
->pbuffer
;
1606 /* use first buffer to collect all the data */
1608 pdata
= pasync_handle
->pbuffer
;
1609 dlen
= pasync_handle
->buffer_len
;
1612 if (!pasync_handle
->buffer_len
||
1613 (dlen
+ pasync_handle
->buffer_len
) >
1614 pasync_ctx
->async_data
.buffer_size
)
1616 memcpy(pdata
+ dlen
, pasync_handle
->pbuffer
,
1617 pasync_handle
->buffer_len
);
1618 dlen
+= pasync_handle
->buffer_len
;
1621 if (!plast_handle
->is_final
) {
1622 /* last handle should have final PDU notification from FW */
1623 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_ISCSI
,
1624 "BM_%d : cid %u %p fwd async PDU opcode %x with last handle missing - HL%u:DN%u:DR%u\n",
1625 beiscsi_conn
->beiscsi_conn_cid
, plast_handle
,
1626 AMAP_GET_BITS(struct amap_pdu_base
, opcode
, phdr
),
1627 pasync_ctx
->async_entry
[cri
].wq
.hdr_len
,
1628 pasync_ctx
->async_entry
[cri
].wq
.bytes_needed
,
1629 pasync_ctx
->async_entry
[cri
].wq
.bytes_received
);
1631 spin_lock_bh(&session
->back_lock
);
1632 status
= beiscsi_complete_pdu(beiscsi_conn
, phdr
, pdata
, dlen
);
1633 spin_unlock_bh(&session
->back_lock
);
1634 beiscsi_hdl_purge_handles(phba
, pasync_ctx
, cri
);
1639 beiscsi_hdl_gather_pdu(struct beiscsi_conn
*beiscsi_conn
,
1640 struct hd_async_context
*pasync_ctx
,
1641 struct hd_async_handle
*pasync_handle
)
1643 unsigned int bytes_needed
= 0, status
= 0;
1644 u16 cri
= pasync_handle
->cri
;
1645 struct cri_wait_queue
*wq
;
1646 struct beiscsi_hba
*phba
;
1647 struct pdu_base
*ppdu
;
1650 phba
= beiscsi_conn
->phba
;
1651 wq
= &pasync_ctx
->async_entry
[cri
].wq
;
1652 if (pasync_handle
->is_header
) {
1653 /* check if PDU hdr is rcv'd when old hdr not completed */
1658 ppdu
= pasync_handle
->pbuffer
;
1659 bytes_needed
= AMAP_GET_BITS(struct amap_pdu_base
,
1661 bytes_needed
<<= 16;
1662 bytes_needed
|= be16_to_cpu(AMAP_GET_BITS(struct amap_pdu_base
,
1663 data_len_lo
, ppdu
));
1664 wq
->hdr_len
= pasync_handle
->buffer_len
;
1665 wq
->bytes_received
= 0;
1666 wq
->bytes_needed
= bytes_needed
;
1667 list_add_tail(&pasync_handle
->link
, &wq
->list
);
1669 status
= beiscsi_hdl_fwd_pdu(beiscsi_conn
,
1672 /* check if data received has header and is needed */
1673 if (!wq
->hdr_len
|| !wq
->bytes_needed
) {
1674 err
= "header less";
1677 wq
->bytes_received
+= pasync_handle
->buffer_len
;
1678 /* Something got overwritten? Better catch it here. */
1679 if (wq
->bytes_received
> wq
->bytes_needed
) {
1683 list_add_tail(&pasync_handle
->link
, &wq
->list
);
1684 if (wq
->bytes_received
== wq
->bytes_needed
)
1685 status
= beiscsi_hdl_fwd_pdu(beiscsi_conn
,
1691 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_ISCSI
,
1692 "BM_%d : cid %u async PDU %s - def-%c:HL%u:DN%u:DR%u\n",
1693 beiscsi_conn
->beiscsi_conn_cid
, err
,
1694 pasync_handle
->is_header
? 'H' : 'D',
1695 wq
->hdr_len
, wq
->bytes_needed
,
1696 pasync_handle
->buffer_len
);
1697 /* discard this handle */
1698 beiscsi_hdl_put_handle(pasync_ctx
, pasync_handle
);
1699 /* free all the other handles in cri_wait_queue */
1700 beiscsi_hdl_purge_handles(phba
, pasync_ctx
, cri
);
1701 /* try continuing */
1706 beiscsi_hdq_post_handles(struct beiscsi_hba
*phba
,
1707 u8 header
, u8 ulp_num
, u16 nbuf
)
1709 struct hd_async_handle
*pasync_handle
;
1710 struct hd_async_context
*pasync_ctx
;
1711 struct hwi_controller
*phwi_ctrlr
;
1712 struct phys_addr
*pasync_sge
;
1713 u32 ring_id
, doorbell
= 0;
1714 u32 doorbell_offset
;
1717 phwi_ctrlr
= phba
->phwi_ctrlr
;
1718 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
, ulp_num
);
1720 pasync_sge
= pasync_ctx
->async_header
.ring_base
;
1721 pi
= pasync_ctx
->async_header
.pi
;
1722 ring_id
= phwi_ctrlr
->default_pdu_hdr
[ulp_num
].id
;
1723 doorbell_offset
= phwi_ctrlr
->default_pdu_hdr
[ulp_num
].
1726 pasync_sge
= pasync_ctx
->async_data
.ring_base
;
1727 pi
= pasync_ctx
->async_data
.pi
;
1728 ring_id
= phwi_ctrlr
->default_pdu_data
[ulp_num
].id
;
1729 doorbell_offset
= phwi_ctrlr
->default_pdu_data
[ulp_num
].
1733 for (prod
= 0; prod
< nbuf
; prod
++) {
1735 pasync_handle
= pasync_ctx
->async_entry
[pi
].header
;
1737 pasync_handle
= pasync_ctx
->async_entry
[pi
].data
;
1738 WARN_ON(pasync_handle
->is_header
!= header
);
1739 WARN_ON(pasync_handle
->index
!= pi
);
1740 /* setup the ring only once */
1741 if (nbuf
== pasync_ctx
->num_entries
) {
1743 pasync_sge
[pi
].hi
= pasync_handle
->pa
.u
.a32
.address_lo
;
1744 pasync_sge
[pi
].lo
= pasync_handle
->pa
.u
.a32
.address_hi
;
1746 if (++pi
== pasync_ctx
->num_entries
)
1751 pasync_ctx
->async_header
.pi
= pi
;
1753 pasync_ctx
->async_data
.pi
= pi
;
1755 doorbell
|= ring_id
& DB_DEF_PDU_RING_ID_MASK
;
1756 doorbell
|= 1 << DB_DEF_PDU_REARM_SHIFT
;
1757 doorbell
|= 0 << DB_DEF_PDU_EVENT_SHIFT
;
1758 doorbell
|= (prod
& DB_DEF_PDU_CQPROC_MASK
) << DB_DEF_PDU_CQPROC_SHIFT
;
1759 iowrite32(doorbell
, phba
->db_va
+ doorbell_offset
);
1763 beiscsi_hdq_process_compl(struct beiscsi_conn
*beiscsi_conn
,
1764 struct i_t_dpdu_cqe
*pdpdu_cqe
)
1766 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
1767 struct hd_async_handle
*pasync_handle
= NULL
;
1768 struct hd_async_context
*pasync_ctx
;
1769 struct hwi_controller
*phwi_ctrlr
;
1770 u8 ulp_num
, consumed
, header
= 0;
1773 phwi_ctrlr
= phba
->phwi_ctrlr
;
1774 cid_cri
= BE_GET_CRI_FROM_CID(beiscsi_conn
->beiscsi_conn_cid
);
1775 ulp_num
= BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr
, cid_cri
);
1776 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
, ulp_num
);
1777 pasync_handle
= beiscsi_hdl_get_handle(beiscsi_conn
, pasync_ctx
,
1778 pdpdu_cqe
, &header
);
1779 if (is_chip_be2_be3r(phba
))
1780 consumed
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe
,
1781 num_cons
, pdpdu_cqe
);
1783 consumed
= AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2
,
1784 num_cons
, pdpdu_cqe
);
1786 beiscsi_hdl_gather_pdu(beiscsi_conn
, pasync_ctx
, pasync_handle
);
1787 /* num_cons indicates number of 8 RQEs consumed */
1789 beiscsi_hdq_post_handles(phba
, header
, ulp_num
, 8 * consumed
);
1792 void beiscsi_process_mcc_cq(struct beiscsi_hba
*phba
)
1794 struct be_queue_info
*mcc_cq
;
1795 struct be_mcc_compl
*mcc_compl
;
1796 unsigned int num_processed
= 0;
1798 mcc_cq
= &phba
->ctrl
.mcc_obj
.cq
;
1799 mcc_compl
= queue_tail_node(mcc_cq
);
1800 mcc_compl
->flags
= le32_to_cpu(mcc_compl
->flags
);
1801 while (mcc_compl
->flags
& CQE_FLAGS_VALID_MASK
) {
1802 if (beiscsi_hba_in_error(phba
))
1805 if (num_processed
>= 32) {
1806 hwi_ring_cq_db(phba
, mcc_cq
->id
,
1810 if (mcc_compl
->flags
& CQE_FLAGS_ASYNC_MASK
) {
1811 beiscsi_process_async_event(phba
, mcc_compl
);
1812 } else if (mcc_compl
->flags
& CQE_FLAGS_COMPLETED_MASK
) {
1813 beiscsi_process_mcc_compl(&phba
->ctrl
, mcc_compl
);
1816 mcc_compl
->flags
= 0;
1817 queue_tail_inc(mcc_cq
);
1818 mcc_compl
= queue_tail_node(mcc_cq
);
1819 mcc_compl
->flags
= le32_to_cpu(mcc_compl
->flags
);
1823 if (num_processed
> 0)
1824 hwi_ring_cq_db(phba
, mcc_cq
->id
, num_processed
, 1);
1827 static void beiscsi_mcc_work(struct work_struct
*work
)
1829 struct be_eq_obj
*pbe_eq
;
1830 struct beiscsi_hba
*phba
;
1832 pbe_eq
= container_of(work
, struct be_eq_obj
, mcc_work
);
1833 phba
= pbe_eq
->phba
;
1834 beiscsi_process_mcc_cq(phba
);
1835 /* rearm EQ for further interrupts */
1836 if (!beiscsi_hba_in_error(phba
))
1837 hwi_ring_eq_db(phba
, pbe_eq
->q
.id
, 0, 0, 1, 1);
1841 * beiscsi_process_cq()- Process the Completion Queue
1842 * @pbe_eq: Event Q on which the Completion has come
1843 * @budget: Max number of events to processed
1846 * Number of Completion Entries processed.
1848 unsigned int beiscsi_process_cq(struct be_eq_obj
*pbe_eq
, int budget
)
1850 struct be_queue_info
*cq
;
1851 struct sol_cqe
*sol
;
1852 unsigned int total
= 0;
1853 unsigned int num_processed
= 0;
1854 unsigned short code
= 0, cid
= 0;
1855 uint16_t cri_index
= 0;
1856 struct beiscsi_conn
*beiscsi_conn
;
1857 struct beiscsi_endpoint
*beiscsi_ep
;
1858 struct iscsi_endpoint
*ep
;
1859 struct beiscsi_hba
*phba
;
1862 sol
= queue_tail_node(cq
);
1863 phba
= pbe_eq
->phba
;
1865 while (sol
->dw
[offsetof(struct amap_sol_cqe
, valid
) / 32] &
1867 if (beiscsi_hba_in_error(phba
))
1870 be_dws_le_to_cpu(sol
, sizeof(struct sol_cqe
));
1872 code
= (sol
->dw
[offsetof(struct amap_sol_cqe
, code
) / 32] &
1876 if (is_chip_be2_be3r(phba
)) {
1877 cid
= AMAP_GET_BITS(struct amap_sol_cqe
, cid
, sol
);
1879 if ((code
== DRIVERMSG_NOTIFY
) ||
1880 (code
== UNSOL_HDR_NOTIFY
) ||
1881 (code
== UNSOL_DATA_NOTIFY
))
1882 cid
= AMAP_GET_BITS(
1883 struct amap_i_t_dpdu_cqe_v2
,
1886 cid
= AMAP_GET_BITS(struct amap_sol_cqe_v2
,
1890 cri_index
= BE_GET_CRI_FROM_CID(cid
);
1891 ep
= phba
->ep_array
[cri_index
];
1894 /* connection has already been freed
1895 * just move on to next one
1897 beiscsi_log(phba
, KERN_WARNING
,
1899 "BM_%d : proc cqe of disconn ep: cid %d\n",
1904 beiscsi_ep
= ep
->dd_data
;
1905 beiscsi_conn
= beiscsi_ep
->conn
;
1908 if (num_processed
== 32) {
1909 hwi_ring_cq_db(phba
, cq
->id
, 32, 0);
1915 case SOL_CMD_COMPLETE
:
1916 hwi_complete_cmd(beiscsi_conn
, phba
, sol
);
1918 case DRIVERMSG_NOTIFY
:
1919 beiscsi_log(phba
, KERN_INFO
,
1920 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
1921 "BM_%d : Received %s[%d] on CID : %d\n",
1922 cqe_desc
[code
], code
, cid
);
1924 hwi_complete_drvr_msgs(beiscsi_conn
, phba
, sol
);
1926 case UNSOL_HDR_NOTIFY
:
1927 beiscsi_log(phba
, KERN_INFO
,
1928 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
1929 "BM_%d : Received %s[%d] on CID : %d\n",
1930 cqe_desc
[code
], code
, cid
);
1932 spin_lock_bh(&phba
->async_pdu_lock
);
1933 beiscsi_hdq_process_compl(beiscsi_conn
,
1934 (struct i_t_dpdu_cqe
*)sol
);
1935 spin_unlock_bh(&phba
->async_pdu_lock
);
1937 case UNSOL_DATA_NOTIFY
:
1938 beiscsi_log(phba
, KERN_INFO
,
1939 BEISCSI_LOG_CONFIG
| BEISCSI_LOG_IO
,
1940 "BM_%d : Received %s[%d] on CID : %d\n",
1941 cqe_desc
[code
], code
, cid
);
1943 spin_lock_bh(&phba
->async_pdu_lock
);
1944 beiscsi_hdq_process_compl(beiscsi_conn
,
1945 (struct i_t_dpdu_cqe
*)sol
);
1946 spin_unlock_bh(&phba
->async_pdu_lock
);
1948 case CXN_INVALIDATE_INDEX_NOTIFY
:
1949 case CMD_INVALIDATED_NOTIFY
:
1950 case CXN_INVALIDATE_NOTIFY
:
1951 beiscsi_log(phba
, KERN_ERR
,
1952 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
1953 "BM_%d : Ignoring %s[%d] on CID : %d\n",
1954 cqe_desc
[code
], code
, cid
);
1956 case CXN_KILLED_HDR_DIGEST_ERR
:
1957 case SOL_CMD_KILLED_DATA_DIGEST_ERR
:
1958 beiscsi_log(phba
, KERN_ERR
,
1959 BEISCSI_LOG_CONFIG
| BEISCSI_LOG_IO
,
1960 "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
1961 cqe_desc
[code
], code
, cid
);
1963 case CMD_KILLED_INVALID_STATSN_RCVD
:
1964 case CMD_KILLED_INVALID_R2T_RCVD
:
1965 case CMD_CXN_KILLED_LUN_INVALID
:
1966 case CMD_CXN_KILLED_ICD_INVALID
:
1967 case CMD_CXN_KILLED_ITT_INVALID
:
1968 case CMD_CXN_KILLED_SEQ_OUTOFORDER
:
1969 case CMD_CXN_KILLED_INVALID_DATASN_RCVD
:
1970 beiscsi_log(phba
, KERN_ERR
,
1971 BEISCSI_LOG_CONFIG
| BEISCSI_LOG_IO
,
1972 "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
1973 cqe_desc
[code
], code
, cid
);
1975 case UNSOL_DATA_DIGEST_ERROR_NOTIFY
:
1976 beiscsi_log(phba
, KERN_ERR
,
1977 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
1978 "BM_%d : Dropping %s[%d] on DPDU ring on CID : %d\n",
1979 cqe_desc
[code
], code
, cid
);
1980 spin_lock_bh(&phba
->async_pdu_lock
);
1981 /* driver consumes the entry and drops the contents */
1982 beiscsi_hdq_process_compl(beiscsi_conn
,
1983 (struct i_t_dpdu_cqe
*)sol
);
1984 spin_unlock_bh(&phba
->async_pdu_lock
);
1986 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL
:
1987 case CXN_KILLED_BURST_LEN_MISMATCH
:
1988 case CXN_KILLED_AHS_RCVD
:
1989 case CXN_KILLED_UNKNOWN_HDR
:
1990 case CXN_KILLED_STALE_ITT_TTT_RCVD
:
1991 case CXN_KILLED_INVALID_ITT_TTT_RCVD
:
1992 case CXN_KILLED_TIMED_OUT
:
1993 case CXN_KILLED_FIN_RCVD
:
1994 case CXN_KILLED_RST_SENT
:
1995 case CXN_KILLED_RST_RCVD
:
1996 case CXN_KILLED_BAD_UNSOL_PDU_RCVD
:
1997 case CXN_KILLED_BAD_WRB_INDEX_ERROR
:
1998 case CXN_KILLED_OVER_RUN_RESIDUAL
:
1999 case CXN_KILLED_UNDER_RUN_RESIDUAL
:
2000 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN
:
2001 beiscsi_log(phba
, KERN_ERR
,
2002 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
2003 "BM_%d : Event %s[%d] received on CID : %d\n",
2004 cqe_desc
[code
], code
, cid
);
2006 iscsi_conn_failure(beiscsi_conn
->conn
,
2007 ISCSI_ERR_CONN_FAILED
);
2010 beiscsi_log(phba
, KERN_ERR
,
2011 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
2012 "BM_%d : Invalid CQE Event Received Code : %d"
2019 AMAP_SET_BITS(struct amap_sol_cqe
, valid
, sol
, 0);
2021 sol
= queue_tail_node(cq
);
2023 if (total
== budget
)
2027 hwi_ring_cq_db(phba
, cq
->id
, num_processed
, 1);
2031 static int be_iopoll(struct irq_poll
*iop
, int budget
)
2033 unsigned int ret
, io_events
;
2034 struct beiscsi_hba
*phba
;
2035 struct be_eq_obj
*pbe_eq
;
2036 struct be_eq_entry
*eqe
= NULL
;
2037 struct be_queue_info
*eq
;
2039 pbe_eq
= container_of(iop
, struct be_eq_obj
, iopoll
);
2040 phba
= pbe_eq
->phba
;
2041 if (beiscsi_hba_in_error(phba
)) {
2042 irq_poll_complete(iop
);
2048 eqe
= queue_tail_node(eq
);
2049 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32] &
2051 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
2053 eqe
= queue_tail_node(eq
);
2056 hwi_ring_eq_db(phba
, eq
->id
, 1, io_events
, 0, 1);
2058 ret
= beiscsi_process_cq(pbe_eq
, budget
);
2059 pbe_eq
->cq_count
+= ret
;
2061 irq_poll_complete(iop
);
2062 beiscsi_log(phba
, KERN_INFO
,
2063 BEISCSI_LOG_CONFIG
| BEISCSI_LOG_IO
,
2064 "BM_%d : rearm pbe_eq->q.id =%d ret %d\n",
2066 if (!beiscsi_hba_in_error(phba
))
2067 hwi_ring_eq_db(phba
, pbe_eq
->q
.id
, 0, 0, 1, 1);
2073 hwi_write_sgl_v2(struct iscsi_wrb
*pwrb
, struct scatterlist
*sg
,
2074 unsigned int num_sg
, struct beiscsi_io_task
*io_task
)
2076 struct iscsi_sge
*psgl
;
2077 unsigned int sg_len
, index
;
2078 unsigned int sge_len
= 0;
2079 unsigned long long addr
;
2080 struct scatterlist
*l_sg
;
2081 unsigned int offset
;
2083 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, iscsi_bhs_addr_lo
, pwrb
,
2084 io_task
->bhs_pa
.u
.a32
.address_lo
);
2085 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, iscsi_bhs_addr_hi
, pwrb
,
2086 io_task
->bhs_pa
.u
.a32
.address_hi
);
2089 for (index
= 0; (index
< num_sg
) && (index
< 2); index
++,
2092 sg_len
= sg_dma_len(sg
);
2093 addr
= (u64
) sg_dma_address(sg
);
2094 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
,
2096 lower_32_bits(addr
));
2097 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
,
2099 upper_32_bits(addr
));
2100 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
,
2105 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sge1_r2t_offset
,
2107 sg_len
= sg_dma_len(sg
);
2108 addr
= (u64
) sg_dma_address(sg
);
2109 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
,
2111 lower_32_bits(addr
));
2112 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
,
2114 upper_32_bits(addr
));
2115 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
,
2120 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
2121 memset(psgl
, 0, sizeof(*psgl
) * BE2_SGE
);
2123 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
- 2);
2125 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2126 io_task
->bhs_pa
.u
.a32
.address_hi
);
2127 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2128 io_task
->bhs_pa
.u
.a32
.address_lo
);
2131 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sge0_last
, pwrb
,
2133 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sge1_last
, pwrb
,
2135 } else if (num_sg
== 2) {
2136 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sge0_last
, pwrb
,
2138 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sge1_last
, pwrb
,
2141 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sge0_last
, pwrb
,
2143 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sge1_last
, pwrb
,
2151 for (index
= 0; index
< num_sg
; index
++, sg
= sg_next(sg
), psgl
++) {
2152 sg_len
= sg_dma_len(sg
);
2153 addr
= (u64
) sg_dma_address(sg
);
2154 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2155 lower_32_bits(addr
));
2156 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2157 upper_32_bits(addr
));
2158 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, sg_len
);
2159 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, offset
);
2160 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
2164 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
2168 hwi_write_sgl(struct iscsi_wrb
*pwrb
, struct scatterlist
*sg
,
2169 unsigned int num_sg
, struct beiscsi_io_task
*io_task
)
2171 struct iscsi_sge
*psgl
;
2172 unsigned int sg_len
, index
;
2173 unsigned int sge_len
= 0;
2174 unsigned long long addr
;
2175 struct scatterlist
*l_sg
;
2176 unsigned int offset
;
2178 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_lo
, pwrb
,
2179 io_task
->bhs_pa
.u
.a32
.address_lo
);
2180 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_hi
, pwrb
,
2181 io_task
->bhs_pa
.u
.a32
.address_hi
);
2184 for (index
= 0; (index
< num_sg
) && (index
< 2); index
++,
2187 sg_len
= sg_dma_len(sg
);
2188 addr
= (u64
) sg_dma_address(sg
);
2189 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_lo
, pwrb
,
2190 ((u32
)(addr
& 0xFFFFFFFF)));
2191 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_hi
, pwrb
,
2192 ((u32
)(addr
>> 32)));
2193 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_len
, pwrb
,
2197 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_r2t_offset
,
2199 sg_len
= sg_dma_len(sg
);
2200 addr
= (u64
) sg_dma_address(sg
);
2201 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_addr_lo
, pwrb
,
2202 ((u32
)(addr
& 0xFFFFFFFF)));
2203 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_addr_hi
, pwrb
,
2204 ((u32
)(addr
>> 32)));
2205 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_len
, pwrb
,
2209 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
2210 memset(psgl
, 0, sizeof(*psgl
) * BE2_SGE
);
2212 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
- 2);
2214 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2215 io_task
->bhs_pa
.u
.a32
.address_hi
);
2216 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2217 io_task
->bhs_pa
.u
.a32
.address_lo
);
2220 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2222 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2224 } else if (num_sg
== 2) {
2225 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2227 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2230 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2232 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2239 for (index
= 0; index
< num_sg
; index
++, sg
= sg_next(sg
), psgl
++) {
2240 sg_len
= sg_dma_len(sg
);
2241 addr
= (u64
) sg_dma_address(sg
);
2242 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2243 (addr
& 0xFFFFFFFF));
2244 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2246 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, sg_len
);
2247 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, offset
);
2248 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
2252 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
2256 * hwi_write_buffer()- Populate the WRB with task info
2257 * @pwrb: ptr to the WRB entry
2258 * @task: iscsi task which is to be executed
2260 static int hwi_write_buffer(struct iscsi_wrb
*pwrb
, struct iscsi_task
*task
)
2262 struct iscsi_sge
*psgl
;
2263 struct beiscsi_io_task
*io_task
= task
->dd_data
;
2264 struct beiscsi_conn
*beiscsi_conn
= io_task
->conn
;
2265 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
2266 uint8_t dsp_value
= 0;
2268 io_task
->bhs_len
= sizeof(struct be_nonio_bhs
) - 2;
2269 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_lo
, pwrb
,
2270 io_task
->bhs_pa
.u
.a32
.address_lo
);
2271 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_hi
, pwrb
,
2272 io_task
->bhs_pa
.u
.a32
.address_hi
);
2276 /* Check for the data_count */
2277 dsp_value
= (task
->data_count
) ? 1 : 0;
2279 if (is_chip_be2_be3r(phba
))
2280 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
,
2283 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, dsp
,
2286 /* Map addr only if there is data_count */
2288 io_task
->mtask_addr
= dma_map_single(&phba
->pcidev
->dev
,
2292 if (dma_mapping_error(&phba
->pcidev
->dev
,
2293 io_task
->mtask_addr
))
2295 io_task
->mtask_data_count
= task
->data_count
;
2297 io_task
->mtask_addr
= 0;
2299 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_lo
, pwrb
,
2300 lower_32_bits(io_task
->mtask_addr
));
2301 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_hi
, pwrb
,
2302 upper_32_bits(io_task
->mtask_addr
));
2303 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_len
, pwrb
,
2306 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
, 1);
2308 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
2309 io_task
->mtask_addr
= 0;
2312 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
2314 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
);
2316 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2317 io_task
->bhs_pa
.u
.a32
.address_hi
);
2318 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2319 io_task
->bhs_pa
.u
.a32
.address_lo
);
2322 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
, 0);
2323 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
, 0);
2324 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, 0);
2325 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, 0);
2326 AMAP_SET_BITS(struct amap_iscsi_sge
, rsvd0
, psgl
, 0);
2327 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
2331 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2332 lower_32_bits(io_task
->mtask_addr
));
2333 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2334 upper_32_bits(io_task
->mtask_addr
));
2336 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, 0x106);
2338 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
2343 * beiscsi_find_mem_req()- Find mem needed
2344 * @phba: ptr to HBA struct
2346 static void beiscsi_find_mem_req(struct beiscsi_hba
*phba
)
2348 uint8_t mem_descr_index
, ulp_num
;
2349 unsigned int num_async_pdu_buf_pages
;
2350 unsigned int num_async_pdu_data_pages
, wrb_sz_per_cxn
;
2351 unsigned int num_async_pdu_buf_sgl_pages
, num_async_pdu_data_sgl_pages
;
2353 phba
->params
.hwi_ws_sz
= sizeof(struct hwi_controller
);
2355 phba
->mem_req
[ISCSI_MEM_GLOBAL_HEADER
] = 2 *
2356 BE_ISCSI_PDU_HEADER_SIZE
;
2357 phba
->mem_req
[HWI_MEM_ADDN_CONTEXT
] =
2358 sizeof(struct hwi_context_memory
);
2361 phba
->mem_req
[HWI_MEM_WRB
] = sizeof(struct iscsi_wrb
)
2362 * (phba
->params
.wrbs_per_cxn
)
2363 * phba
->params
.cxns_per_ctrl
;
2364 wrb_sz_per_cxn
= sizeof(struct wrb_handle
) *
2365 (phba
->params
.wrbs_per_cxn
);
2366 phba
->mem_req
[HWI_MEM_WRBH
] = roundup_pow_of_two((wrb_sz_per_cxn
) *
2367 phba
->params
.cxns_per_ctrl
);
2369 phba
->mem_req
[HWI_MEM_SGLH
] = sizeof(struct sgl_handle
) *
2370 phba
->params
.icds_per_ctrl
;
2371 phba
->mem_req
[HWI_MEM_SGE
] = sizeof(struct iscsi_sge
) *
2372 phba
->params
.num_sge_per_io
* phba
->params
.icds_per_ctrl
;
2373 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
2374 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
)) {
2376 num_async_pdu_buf_sgl_pages
=
2377 PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE(
2379 sizeof(struct phys_addr
));
2381 num_async_pdu_buf_pages
=
2382 PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE(
2384 phba
->params
.defpdu_hdr_sz
);
2386 num_async_pdu_data_pages
=
2387 PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE(
2389 phba
->params
.defpdu_data_sz
);
2391 num_async_pdu_data_sgl_pages
=
2392 PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE(
2394 sizeof(struct phys_addr
));
2396 mem_descr_index
= (HWI_MEM_TEMPLATE_HDR_ULP0
+
2397 (ulp_num
* MEM_DESCR_OFFSET
));
2398 phba
->mem_req
[mem_descr_index
] =
2399 BEISCSI_GET_CID_COUNT(phba
, ulp_num
) *
2400 BEISCSI_TEMPLATE_HDR_PER_CXN_SIZE
;
2402 mem_descr_index
= (HWI_MEM_ASYNC_HEADER_BUF_ULP0
+
2403 (ulp_num
* MEM_DESCR_OFFSET
));
2404 phba
->mem_req
[mem_descr_index
] =
2405 num_async_pdu_buf_pages
*
2408 mem_descr_index
= (HWI_MEM_ASYNC_DATA_BUF_ULP0
+
2409 (ulp_num
* MEM_DESCR_OFFSET
));
2410 phba
->mem_req
[mem_descr_index
] =
2411 num_async_pdu_data_pages
*
2414 mem_descr_index
= (HWI_MEM_ASYNC_HEADER_RING_ULP0
+
2415 (ulp_num
* MEM_DESCR_OFFSET
));
2416 phba
->mem_req
[mem_descr_index
] =
2417 num_async_pdu_buf_sgl_pages
*
2420 mem_descr_index
= (HWI_MEM_ASYNC_DATA_RING_ULP0
+
2421 (ulp_num
* MEM_DESCR_OFFSET
));
2422 phba
->mem_req
[mem_descr_index
] =
2423 num_async_pdu_data_sgl_pages
*
2426 mem_descr_index
= (HWI_MEM_ASYNC_HEADER_HANDLE_ULP0
+
2427 (ulp_num
* MEM_DESCR_OFFSET
));
2428 phba
->mem_req
[mem_descr_index
] =
2429 BEISCSI_ASYNC_HDQ_SIZE(phba
, ulp_num
) *
2430 sizeof(struct hd_async_handle
);
2432 mem_descr_index
= (HWI_MEM_ASYNC_DATA_HANDLE_ULP0
+
2433 (ulp_num
* MEM_DESCR_OFFSET
));
2434 phba
->mem_req
[mem_descr_index
] =
2435 BEISCSI_ASYNC_HDQ_SIZE(phba
, ulp_num
) *
2436 sizeof(struct hd_async_handle
);
2438 mem_descr_index
= (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0
+
2439 (ulp_num
* MEM_DESCR_OFFSET
));
2440 phba
->mem_req
[mem_descr_index
] =
2441 sizeof(struct hd_async_context
) +
2442 (BEISCSI_ASYNC_HDQ_SIZE(phba
, ulp_num
) *
2443 sizeof(struct hd_async_entry
));
2448 static int beiscsi_alloc_mem(struct beiscsi_hba
*phba
)
2451 struct hwi_controller
*phwi_ctrlr
;
2452 struct be_mem_descriptor
*mem_descr
;
2453 struct mem_array
*mem_arr
, *mem_arr_orig
;
2454 unsigned int i
, j
, alloc_size
, curr_alloc_size
;
2456 phba
->phwi_ctrlr
= kzalloc(phba
->params
.hwi_ws_sz
, GFP_KERNEL
);
2457 if (!phba
->phwi_ctrlr
)
2460 /* Allocate memory for wrb_context */
2461 phwi_ctrlr
= phba
->phwi_ctrlr
;
2462 phwi_ctrlr
->wrb_context
= kcalloc(phba
->params
.cxns_per_ctrl
,
2463 sizeof(struct hwi_wrb_context
),
2465 if (!phwi_ctrlr
->wrb_context
) {
2466 kfree(phba
->phwi_ctrlr
);
2470 phba
->init_mem
= kcalloc(SE_MEM_MAX
, sizeof(*mem_descr
),
2472 if (!phba
->init_mem
) {
2473 kfree(phwi_ctrlr
->wrb_context
);
2474 kfree(phba
->phwi_ctrlr
);
2478 mem_arr_orig
= kmalloc_array(BEISCSI_MAX_FRAGS_INIT
,
2479 sizeof(*mem_arr_orig
),
2481 if (!mem_arr_orig
) {
2482 kfree(phba
->init_mem
);
2483 kfree(phwi_ctrlr
->wrb_context
);
2484 kfree(phba
->phwi_ctrlr
);
2488 mem_descr
= phba
->init_mem
;
2489 for (i
= 0; i
< SE_MEM_MAX
; i
++) {
2490 if (!phba
->mem_req
[i
]) {
2491 mem_descr
->mem_array
= NULL
;
2497 mem_arr
= mem_arr_orig
;
2498 alloc_size
= phba
->mem_req
[i
];
2499 memset(mem_arr
, 0, sizeof(struct mem_array
) *
2500 BEISCSI_MAX_FRAGS_INIT
);
2501 curr_alloc_size
= min(be_max_phys_size
* 1024, alloc_size
);
2503 mem_arr
->virtual_address
=
2504 dma_alloc_coherent(&phba
->pcidev
->dev
,
2505 curr_alloc_size
, &bus_add
, GFP_KERNEL
);
2506 if (!mem_arr
->virtual_address
) {
2507 if (curr_alloc_size
<= BE_MIN_MEM_SIZE
)
2509 if (curr_alloc_size
-
2510 rounddown_pow_of_two(curr_alloc_size
))
2511 curr_alloc_size
= rounddown_pow_of_two
2514 curr_alloc_size
= curr_alloc_size
/ 2;
2516 mem_arr
->bus_address
.u
.
2517 a64
.address
= (__u64
) bus_add
;
2518 mem_arr
->size
= curr_alloc_size
;
2519 alloc_size
-= curr_alloc_size
;
2520 curr_alloc_size
= min(be_max_phys_size
*
2525 } while (alloc_size
);
2526 mem_descr
->num_elements
= j
;
2527 mem_descr
->size_in_bytes
= phba
->mem_req
[i
];
2528 mem_descr
->mem_array
= kmalloc_array(j
, sizeof(*mem_arr
),
2530 if (!mem_descr
->mem_array
)
2533 memcpy(mem_descr
->mem_array
, mem_arr_orig
,
2534 sizeof(struct mem_array
) * j
);
2537 kfree(mem_arr_orig
);
2540 mem_descr
->num_elements
= j
;
2541 while ((i
) || (j
)) {
2542 for (j
= mem_descr
->num_elements
; j
> 0; j
--) {
2543 dma_free_coherent(&phba
->pcidev
->dev
,
2544 mem_descr
->mem_array
[j
- 1].size
,
2545 mem_descr
->mem_array
[j
- 1].
2547 (unsigned long)mem_descr
->
2549 bus_address
.u
.a64
.address
);
2553 kfree(mem_descr
->mem_array
);
2557 kfree(mem_arr_orig
);
2558 kfree(phba
->init_mem
);
2559 kfree(phba
->phwi_ctrlr
->wrb_context
);
2560 kfree(phba
->phwi_ctrlr
);
2564 static int beiscsi_get_memory(struct beiscsi_hba
*phba
)
2566 beiscsi_find_mem_req(phba
);
2567 return beiscsi_alloc_mem(phba
);
2570 static void iscsi_init_global_templates(struct beiscsi_hba
*phba
)
2572 struct pdu_data_out
*pdata_out
;
2573 struct pdu_nop_out
*pnop_out
;
2574 struct be_mem_descriptor
*mem_descr
;
2576 mem_descr
= phba
->init_mem
;
2577 mem_descr
+= ISCSI_MEM_GLOBAL_HEADER
;
2579 (struct pdu_data_out
*)mem_descr
->mem_array
[0].virtual_address
;
2580 memset(pdata_out
, 0, BE_ISCSI_PDU_HEADER_SIZE
);
2582 AMAP_SET_BITS(struct amap_pdu_data_out
, opcode
, pdata_out
,
2586 (struct pdu_nop_out
*)((unsigned char *)mem_descr
->mem_array
[0].
2587 virtual_address
+ BE_ISCSI_PDU_HEADER_SIZE
);
2589 memset(pnop_out
, 0, BE_ISCSI_PDU_HEADER_SIZE
);
2590 AMAP_SET_BITS(struct amap_pdu_nop_out
, ttt
, pnop_out
, 0xFFFFFFFF);
2591 AMAP_SET_BITS(struct amap_pdu_nop_out
, f_bit
, pnop_out
, 1);
2592 AMAP_SET_BITS(struct amap_pdu_nop_out
, i_bit
, pnop_out
, 0);
2595 static int beiscsi_init_wrb_handle(struct beiscsi_hba
*phba
)
2597 struct be_mem_descriptor
*mem_descr_wrbh
, *mem_descr_wrb
;
2598 struct hwi_context_memory
*phwi_ctxt
;
2599 struct wrb_handle
*pwrb_handle
= NULL
;
2600 struct hwi_controller
*phwi_ctrlr
;
2601 struct hwi_wrb_context
*pwrb_context
;
2602 struct iscsi_wrb
*pwrb
= NULL
;
2603 unsigned int num_cxn_wrbh
= 0;
2604 unsigned int num_cxn_wrb
= 0, j
, idx
= 0, index
;
2606 mem_descr_wrbh
= phba
->init_mem
;
2607 mem_descr_wrbh
+= HWI_MEM_WRBH
;
2609 mem_descr_wrb
= phba
->init_mem
;
2610 mem_descr_wrb
+= HWI_MEM_WRB
;
2611 phwi_ctrlr
= phba
->phwi_ctrlr
;
2613 /* Allocate memory for WRBQ */
2614 phwi_ctxt
= phwi_ctrlr
->phwi_ctxt
;
2615 phwi_ctxt
->be_wrbq
= kcalloc(phba
->params
.cxns_per_ctrl
,
2616 sizeof(struct be_queue_info
),
2618 if (!phwi_ctxt
->be_wrbq
) {
2619 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
2620 "BM_%d : WRBQ Mem Alloc Failed\n");
2624 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
; index
++) {
2625 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2626 pwrb_context
->pwrb_handle_base
=
2627 kcalloc(phba
->params
.wrbs_per_cxn
,
2628 sizeof(struct wrb_handle
*),
2630 if (!pwrb_context
->pwrb_handle_base
) {
2631 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
2632 "BM_%d : Mem Alloc Failed. Failing to load\n");
2633 goto init_wrb_hndl_failed
;
2635 pwrb_context
->pwrb_handle_basestd
=
2636 kcalloc(phba
->params
.wrbs_per_cxn
,
2637 sizeof(struct wrb_handle
*),
2639 if (!pwrb_context
->pwrb_handle_basestd
) {
2640 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
2641 "BM_%d : Mem Alloc Failed. Failing to load\n");
2642 goto init_wrb_hndl_failed
;
2644 if (!num_cxn_wrbh
) {
2646 mem_descr_wrbh
->mem_array
[idx
].virtual_address
;
2647 num_cxn_wrbh
= ((mem_descr_wrbh
->mem_array
[idx
].size
) /
2648 ((sizeof(struct wrb_handle
)) *
2649 phba
->params
.wrbs_per_cxn
));
2652 pwrb_context
->alloc_index
= 0;
2653 pwrb_context
->wrb_handles_available
= 0;
2654 pwrb_context
->free_index
= 0;
2657 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2658 pwrb_context
->pwrb_handle_base
[j
] = pwrb_handle
;
2659 pwrb_context
->pwrb_handle_basestd
[j
] =
2661 pwrb_context
->wrb_handles_available
++;
2662 pwrb_handle
->wrb_index
= j
;
2667 spin_lock_init(&pwrb_context
->wrb_lock
);
2670 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
; index
++) {
2671 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2673 pwrb
= mem_descr_wrb
->mem_array
[idx
].virtual_address
;
2674 num_cxn_wrb
= (mem_descr_wrb
->mem_array
[idx
].size
) /
2675 ((sizeof(struct iscsi_wrb
) *
2676 phba
->params
.wrbs_per_cxn
));
2681 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2682 pwrb_handle
= pwrb_context
->pwrb_handle_base
[j
];
2683 pwrb_handle
->pwrb
= pwrb
;
2690 init_wrb_hndl_failed
:
2691 for (j
= index
; j
> 0; j
--) {
2692 pwrb_context
= &phwi_ctrlr
->wrb_context
[j
];
2693 kfree(pwrb_context
->pwrb_handle_base
);
2694 kfree(pwrb_context
->pwrb_handle_basestd
);
2699 static int hwi_init_async_pdu_ctx(struct beiscsi_hba
*phba
)
2702 struct hwi_controller
*phwi_ctrlr
;
2703 struct hba_parameters
*p
= &phba
->params
;
2704 struct hd_async_context
*pasync_ctx
;
2705 struct hd_async_handle
*pasync_header_h
, *pasync_data_h
;
2706 unsigned int index
, idx
, num_per_mem
, num_async_data
;
2707 struct be_mem_descriptor
*mem_descr
;
2709 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
2710 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
)) {
2711 /* get async_ctx for each ULP */
2712 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2713 mem_descr
+= (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0
+
2714 (ulp_num
* MEM_DESCR_OFFSET
));
2716 phwi_ctrlr
= phba
->phwi_ctrlr
;
2717 phwi_ctrlr
->phwi_ctxt
->pasync_ctx
[ulp_num
] =
2718 (struct hd_async_context
*)
2719 mem_descr
->mem_array
[0].virtual_address
;
2721 pasync_ctx
= phwi_ctrlr
->phwi_ctxt
->pasync_ctx
[ulp_num
];
2722 memset(pasync_ctx
, 0, sizeof(*pasync_ctx
));
2724 pasync_ctx
->async_entry
=
2725 (struct hd_async_entry
*)
2726 ((long unsigned int)pasync_ctx
+
2727 sizeof(struct hd_async_context
));
2729 pasync_ctx
->num_entries
= BEISCSI_ASYNC_HDQ_SIZE(phba
,
2731 /* setup header buffers */
2732 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2733 mem_descr
+= HWI_MEM_ASYNC_HEADER_BUF_ULP0
+
2734 (ulp_num
* MEM_DESCR_OFFSET
);
2735 if (mem_descr
->mem_array
[0].virtual_address
) {
2736 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
2737 "BM_%d : hwi_init_async_pdu_ctx"
2738 " HWI_MEM_ASYNC_HEADER_BUF_ULP%d va=%p\n",
2740 mem_descr
->mem_array
[0].
2743 beiscsi_log(phba
, KERN_WARNING
,
2745 "BM_%d : No Virtual address for ULP : %d\n",
2748 pasync_ctx
->async_header
.pi
= 0;
2749 pasync_ctx
->async_header
.buffer_size
= p
->defpdu_hdr_sz
;
2750 pasync_ctx
->async_header
.va_base
=
2751 mem_descr
->mem_array
[0].virtual_address
;
2753 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
=
2754 mem_descr
->mem_array
[0].
2755 bus_address
.u
.a64
.address
;
2757 /* setup header buffer sgls */
2758 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2759 mem_descr
+= HWI_MEM_ASYNC_HEADER_RING_ULP0
+
2760 (ulp_num
* MEM_DESCR_OFFSET
);
2761 if (mem_descr
->mem_array
[0].virtual_address
) {
2762 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
2763 "BM_%d : hwi_init_async_pdu_ctx"
2764 " HWI_MEM_ASYNC_HEADER_RING_ULP%d va=%p\n",
2766 mem_descr
->mem_array
[0].
2769 beiscsi_log(phba
, KERN_WARNING
,
2771 "BM_%d : No Virtual address for ULP : %d\n",
2774 pasync_ctx
->async_header
.ring_base
=
2775 mem_descr
->mem_array
[0].virtual_address
;
2777 /* setup header buffer handles */
2778 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2779 mem_descr
+= HWI_MEM_ASYNC_HEADER_HANDLE_ULP0
+
2780 (ulp_num
* MEM_DESCR_OFFSET
);
2781 if (mem_descr
->mem_array
[0].virtual_address
) {
2782 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
2783 "BM_%d : hwi_init_async_pdu_ctx"
2784 " HWI_MEM_ASYNC_HEADER_HANDLE_ULP%d va=%p\n",
2786 mem_descr
->mem_array
[0].
2789 beiscsi_log(phba
, KERN_WARNING
,
2791 "BM_%d : No Virtual address for ULP : %d\n",
2794 pasync_ctx
->async_header
.handle_base
=
2795 mem_descr
->mem_array
[0].virtual_address
;
2797 /* setup data buffer sgls */
2798 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2799 mem_descr
+= HWI_MEM_ASYNC_DATA_RING_ULP0
+
2800 (ulp_num
* MEM_DESCR_OFFSET
);
2801 if (mem_descr
->mem_array
[0].virtual_address
) {
2802 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
2803 "BM_%d : hwi_init_async_pdu_ctx"
2804 " HWI_MEM_ASYNC_DATA_RING_ULP%d va=%p\n",
2806 mem_descr
->mem_array
[0].
2809 beiscsi_log(phba
, KERN_WARNING
,
2811 "BM_%d : No Virtual address for ULP : %d\n",
2814 pasync_ctx
->async_data
.ring_base
=
2815 mem_descr
->mem_array
[0].virtual_address
;
2817 /* setup data buffer handles */
2818 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2819 mem_descr
+= HWI_MEM_ASYNC_DATA_HANDLE_ULP0
+
2820 (ulp_num
* MEM_DESCR_OFFSET
);
2821 if (!mem_descr
->mem_array
[0].virtual_address
)
2822 beiscsi_log(phba
, KERN_WARNING
,
2824 "BM_%d : No Virtual address for ULP : %d\n",
2827 pasync_ctx
->async_data
.handle_base
=
2828 mem_descr
->mem_array
[0].virtual_address
;
2831 (struct hd_async_handle
*)
2832 pasync_ctx
->async_header
.handle_base
;
2834 (struct hd_async_handle
*)
2835 pasync_ctx
->async_data
.handle_base
;
2837 /* setup data buffers */
2838 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2839 mem_descr
+= HWI_MEM_ASYNC_DATA_BUF_ULP0
+
2840 (ulp_num
* MEM_DESCR_OFFSET
);
2841 if (mem_descr
->mem_array
[0].virtual_address
) {
2842 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
2843 "BM_%d : hwi_init_async_pdu_ctx"
2844 " HWI_MEM_ASYNC_DATA_BUF_ULP%d va=%p\n",
2846 mem_descr
->mem_array
[0].
2849 beiscsi_log(phba
, KERN_WARNING
,
2851 "BM_%d : No Virtual address for ULP : %d\n",
2855 pasync_ctx
->async_data
.pi
= 0;
2856 pasync_ctx
->async_data
.buffer_size
= p
->defpdu_data_sz
;
2857 pasync_ctx
->async_data
.va_base
=
2858 mem_descr
->mem_array
[idx
].virtual_address
;
2859 pasync_ctx
->async_data
.pa_base
.u
.a64
.address
=
2860 mem_descr
->mem_array
[idx
].
2861 bus_address
.u
.a64
.address
;
2863 num_async_data
= ((mem_descr
->mem_array
[idx
].size
) /
2864 phba
->params
.defpdu_data_sz
);
2867 for (index
= 0; index
< BEISCSI_ASYNC_HDQ_SIZE
2868 (phba
, ulp_num
); index
++) {
2869 pasync_header_h
->cri
= -1;
2870 pasync_header_h
->is_header
= 1;
2871 pasync_header_h
->index
= index
;
2872 INIT_LIST_HEAD(&pasync_header_h
->link
);
2873 pasync_header_h
->pbuffer
=
2874 (void *)((unsigned long)
2876 async_header
.va_base
) +
2877 (p
->defpdu_hdr_sz
* index
));
2879 pasync_header_h
->pa
.u
.a64
.address
=
2880 pasync_ctx
->async_header
.pa_base
.u
.a64
.
2881 address
+ (p
->defpdu_hdr_sz
* index
);
2883 pasync_ctx
->async_entry
[index
].header
=
2886 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].
2889 pasync_data_h
->cri
= -1;
2890 pasync_data_h
->is_header
= 0;
2891 pasync_data_h
->index
= index
;
2892 INIT_LIST_HEAD(&pasync_data_h
->link
);
2894 if (!num_async_data
) {
2897 pasync_ctx
->async_data
.va_base
=
2898 mem_descr
->mem_array
[idx
].
2900 pasync_ctx
->async_data
.pa_base
.u
.
2902 mem_descr
->mem_array
[idx
].
2903 bus_address
.u
.a64
.address
;
2905 ((mem_descr
->mem_array
[idx
].
2907 phba
->params
.defpdu_data_sz
);
2909 pasync_data_h
->pbuffer
=
2910 (void *)((unsigned long)
2911 (pasync_ctx
->async_data
.va_base
) +
2912 (p
->defpdu_data_sz
* num_per_mem
));
2914 pasync_data_h
->pa
.u
.a64
.address
=
2915 pasync_ctx
->async_data
.pa_base
.u
.a64
.
2916 address
+ (p
->defpdu_data_sz
*
2921 pasync_ctx
->async_entry
[index
].data
=
2932 be_sgl_create_contiguous(void *virtual_address
,
2933 u64 physical_address
, u32 length
,
2934 struct be_dma_mem
*sgl
)
2936 WARN_ON(!virtual_address
);
2937 WARN_ON(!physical_address
);
2941 sgl
->va
= virtual_address
;
2942 sgl
->dma
= (unsigned long)physical_address
;
2948 static void be_sgl_destroy_contiguous(struct be_dma_mem
*sgl
)
2950 memset(sgl
, 0, sizeof(*sgl
));
2954 hwi_build_be_sgl_arr(struct beiscsi_hba
*phba
,
2955 struct mem_array
*pmem
, struct be_dma_mem
*sgl
)
2958 be_sgl_destroy_contiguous(sgl
);
2960 be_sgl_create_contiguous(pmem
->virtual_address
,
2961 pmem
->bus_address
.u
.a64
.address
,
2966 hwi_build_be_sgl_by_offset(struct beiscsi_hba
*phba
,
2967 struct mem_array
*pmem
, struct be_dma_mem
*sgl
)
2970 be_sgl_destroy_contiguous(sgl
);
2972 be_sgl_create_contiguous((unsigned char *)pmem
->virtual_address
,
2973 pmem
->bus_address
.u
.a64
.address
,
2977 static int be_fill_queue(struct be_queue_info
*q
,
2978 u16 len
, u16 entry_size
, void *vaddress
)
2980 struct be_dma_mem
*mem
= &q
->dma_mem
;
2982 memset(q
, 0, sizeof(*q
));
2984 q
->entry_size
= entry_size
;
2985 mem
->size
= len
* entry_size
;
2989 memset(mem
->va
, 0, mem
->size
);
2993 static int beiscsi_create_eqs(struct beiscsi_hba
*phba
,
2994 struct hwi_context_memory
*phwi_context
)
2996 int ret
= -ENOMEM
, eq_for_mcc
;
2997 unsigned int i
, num_eq_pages
;
2998 struct be_queue_info
*eq
;
2999 struct be_dma_mem
*mem
;
3003 num_eq_pages
= PAGES_REQUIRED(phba
->params
.num_eq_entries
* \
3004 sizeof(struct be_eq_entry
));
3006 if (phba
->pcidev
->msix_enabled
)
3010 for (i
= 0; i
< (phba
->num_cpus
+ eq_for_mcc
); i
++) {
3011 eq
= &phwi_context
->be_eq
[i
].q
;
3013 phwi_context
->be_eq
[i
].phba
= phba
;
3014 eq_vaddress
= dma_alloc_coherent(&phba
->pcidev
->dev
,
3015 num_eq_pages
* PAGE_SIZE
,
3016 &paddr
, GFP_KERNEL
);
3019 goto create_eq_error
;
3022 mem
->va
= eq_vaddress
;
3023 ret
= be_fill_queue(eq
, phba
->params
.num_eq_entries
,
3024 sizeof(struct be_eq_entry
), eq_vaddress
);
3026 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3027 "BM_%d : be_fill_queue Failed for EQ\n");
3028 goto create_eq_error
;
3032 ret
= beiscsi_cmd_eq_create(&phba
->ctrl
, eq
,
3033 BEISCSI_EQ_DELAY_DEF
);
3035 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3036 "BM_%d : beiscsi_cmd_eq_create"
3038 goto create_eq_error
;
3041 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3042 "BM_%d : eqid = %d\n",
3043 phwi_context
->be_eq
[i
].q
.id
);
3048 for (i
= 0; i
< (phba
->num_cpus
+ eq_for_mcc
); i
++) {
3049 eq
= &phwi_context
->be_eq
[i
].q
;
3052 dma_free_coherent(&phba
->pcidev
->dev
, num_eq_pages
3059 static int beiscsi_create_cqs(struct beiscsi_hba
*phba
,
3060 struct hwi_context_memory
*phwi_context
)
3062 unsigned int i
, num_cq_pages
;
3063 struct be_queue_info
*cq
, *eq
;
3064 struct be_dma_mem
*mem
;
3065 struct be_eq_obj
*pbe_eq
;
3070 num_cq_pages
= PAGES_REQUIRED(phba
->params
.num_cq_entries
* \
3071 sizeof(struct sol_cqe
));
3073 for (i
= 0; i
< phba
->num_cpus
; i
++) {
3074 cq
= &phwi_context
->be_cq
[i
];
3075 eq
= &phwi_context
->be_eq
[i
].q
;
3076 pbe_eq
= &phwi_context
->be_eq
[i
];
3078 pbe_eq
->phba
= phba
;
3080 cq_vaddress
= dma_alloc_coherent(&phba
->pcidev
->dev
,
3081 num_cq_pages
* PAGE_SIZE
,
3082 &paddr
, GFP_KERNEL
);
3085 goto create_cq_error
;
3088 ret
= be_fill_queue(cq
, phba
->params
.num_cq_entries
,
3089 sizeof(struct sol_cqe
), cq_vaddress
);
3091 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3092 "BM_%d : be_fill_queue Failed "
3094 goto create_cq_error
;
3098 ret
= beiscsi_cmd_cq_create(&phba
->ctrl
, cq
, eq
, false,
3101 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3102 "BM_%d : beiscsi_cmd_eq_create"
3103 "Failed for ISCSI CQ\n");
3104 goto create_cq_error
;
3106 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3107 "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3108 "iSCSI CQ CREATED\n", cq
->id
, eq
->id
);
3113 for (i
= 0; i
< phba
->num_cpus
; i
++) {
3114 cq
= &phwi_context
->be_cq
[i
];
3117 dma_free_coherent(&phba
->pcidev
->dev
, num_cq_pages
3125 beiscsi_create_def_hdr(struct beiscsi_hba
*phba
,
3126 struct hwi_context_memory
*phwi_context
,
3127 struct hwi_controller
*phwi_ctrlr
,
3128 unsigned int def_pdu_ring_sz
, uint8_t ulp_num
)
3132 struct be_queue_info
*dq
, *cq
;
3133 struct be_dma_mem
*mem
;
3134 struct be_mem_descriptor
*mem_descr
;
3138 dq
= &phwi_context
->be_def_hdrq
[ulp_num
];
3139 cq
= &phwi_context
->be_cq
[0];
3141 mem_descr
= phba
->init_mem
;
3142 mem_descr
+= HWI_MEM_ASYNC_HEADER_RING_ULP0
+
3143 (ulp_num
* MEM_DESCR_OFFSET
);
3144 dq_vaddress
= mem_descr
->mem_array
[idx
].virtual_address
;
3145 ret
= be_fill_queue(dq
, mem_descr
->mem_array
[0].size
/
3146 sizeof(struct phys_addr
),
3147 sizeof(struct phys_addr
), dq_vaddress
);
3149 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3150 "BM_%d : be_fill_queue Failed for DEF PDU HDR on ULP : %d\n",
3155 mem
->dma
= (unsigned long)mem_descr
->mem_array
[idx
].
3156 bus_address
.u
.a64
.address
;
3157 ret
= be_cmd_create_default_pdu_queue(&phba
->ctrl
, cq
, dq
,
3159 phba
->params
.defpdu_hdr_sz
,
3160 BEISCSI_DEFQ_HDR
, ulp_num
);
3162 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3163 "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR on ULP : %d\n",
3169 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3170 "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n",
3172 phwi_context
->be_def_hdrq
[ulp_num
].id
);
3177 beiscsi_create_def_data(struct beiscsi_hba
*phba
,
3178 struct hwi_context_memory
*phwi_context
,
3179 struct hwi_controller
*phwi_ctrlr
,
3180 unsigned int def_pdu_ring_sz
, uint8_t ulp_num
)
3184 struct be_queue_info
*dataq
, *cq
;
3185 struct be_dma_mem
*mem
;
3186 struct be_mem_descriptor
*mem_descr
;
3190 dataq
= &phwi_context
->be_def_dataq
[ulp_num
];
3191 cq
= &phwi_context
->be_cq
[0];
3192 mem
= &dataq
->dma_mem
;
3193 mem_descr
= phba
->init_mem
;
3194 mem_descr
+= HWI_MEM_ASYNC_DATA_RING_ULP0
+
3195 (ulp_num
* MEM_DESCR_OFFSET
);
3196 dq_vaddress
= mem_descr
->mem_array
[idx
].virtual_address
;
3197 ret
= be_fill_queue(dataq
, mem_descr
->mem_array
[0].size
/
3198 sizeof(struct phys_addr
),
3199 sizeof(struct phys_addr
), dq_vaddress
);
3201 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3202 "BM_%d : be_fill_queue Failed for DEF PDU "
3203 "DATA on ULP : %d\n",
3208 mem
->dma
= (unsigned long)mem_descr
->mem_array
[idx
].
3209 bus_address
.u
.a64
.address
;
3210 ret
= be_cmd_create_default_pdu_queue(&phba
->ctrl
, cq
, dataq
,
3212 phba
->params
.defpdu_data_sz
,
3213 BEISCSI_DEFQ_DATA
, ulp_num
);
3215 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3216 "BM_%d be_cmd_create_default_pdu_queue"
3217 " Failed for DEF PDU DATA on ULP : %d\n",
3222 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3223 "BM_%d : iscsi def data id on ULP : %d is %d\n",
3225 phwi_context
->be_def_dataq
[ulp_num
].id
);
3227 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3228 "BM_%d : DEFAULT PDU DATA RING CREATED"
3229 "on ULP : %d\n", ulp_num
);
3235 beiscsi_post_template_hdr(struct beiscsi_hba
*phba
)
3237 struct be_mem_descriptor
*mem_descr
;
3238 struct mem_array
*pm_arr
;
3239 struct be_dma_mem sgl
;
3240 int status
, ulp_num
;
3242 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
3243 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
)) {
3244 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
3245 mem_descr
+= HWI_MEM_TEMPLATE_HDR_ULP0
+
3246 (ulp_num
* MEM_DESCR_OFFSET
);
3247 pm_arr
= mem_descr
->mem_array
;
3249 hwi_build_be_sgl_arr(phba
, pm_arr
, &sgl
);
3250 status
= be_cmd_iscsi_post_template_hdr(
3254 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3255 "BM_%d : Post Template HDR Failed for"
3256 "ULP_%d\n", ulp_num
);
3260 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3261 "BM_%d : Template HDR Pages Posted for"
3262 "ULP_%d\n", ulp_num
);
3269 beiscsi_post_pages(struct beiscsi_hba
*phba
)
3271 struct be_mem_descriptor
*mem_descr
;
3272 struct mem_array
*pm_arr
;
3273 unsigned int page_offset
, i
;
3274 struct be_dma_mem sgl
;
3275 int status
, ulp_num
= 0;
3277 mem_descr
= phba
->init_mem
;
3278 mem_descr
+= HWI_MEM_SGE
;
3279 pm_arr
= mem_descr
->mem_array
;
3281 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++)
3282 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
))
3285 page_offset
= (sizeof(struct iscsi_sge
) * phba
->params
.num_sge_per_io
*
3286 phba
->fw_config
.iscsi_icd_start
[ulp_num
]) / PAGE_SIZE
;
3287 for (i
= 0; i
< mem_descr
->num_elements
; i
++) {
3288 hwi_build_be_sgl_arr(phba
, pm_arr
, &sgl
);
3289 status
= be_cmd_iscsi_post_sgl_pages(&phba
->ctrl
, &sgl
,
3291 (pm_arr
->size
/ PAGE_SIZE
));
3292 page_offset
+= pm_arr
->size
/ PAGE_SIZE
;
3294 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3295 "BM_%d : post sgl failed.\n");
3300 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3301 "BM_%d : POSTED PAGES\n");
3305 static void be_queue_free(struct beiscsi_hba
*phba
, struct be_queue_info
*q
)
3307 struct be_dma_mem
*mem
= &q
->dma_mem
;
3309 dma_free_coherent(&phba
->pcidev
->dev
, mem
->size
,
3315 static int be_queue_alloc(struct beiscsi_hba
*phba
, struct be_queue_info
*q
,
3316 u16 len
, u16 entry_size
)
3318 struct be_dma_mem
*mem
= &q
->dma_mem
;
3320 memset(q
, 0, sizeof(*q
));
3322 q
->entry_size
= entry_size
;
3323 mem
->size
= len
* entry_size
;
3324 mem
->va
= dma_alloc_coherent(&phba
->pcidev
->dev
, mem
->size
, &mem
->dma
,
3332 beiscsi_create_wrb_rings(struct beiscsi_hba
*phba
,
3333 struct hwi_context_memory
*phwi_context
,
3334 struct hwi_controller
*phwi_ctrlr
)
3336 unsigned int num_wrb_rings
;
3338 unsigned int idx
, num
, i
, ulp_num
;
3339 struct mem_array
*pwrb_arr
;
3341 struct be_dma_mem sgl
;
3342 struct be_mem_descriptor
*mem_descr
;
3343 struct hwi_wrb_context
*pwrb_context
;
3345 uint8_t ulp_count
= 0, ulp_base_num
= 0;
3346 uint16_t cid_count_ulp
[BEISCSI_ULP_COUNT
] = { 0 };
3349 mem_descr
= phba
->init_mem
;
3350 mem_descr
+= HWI_MEM_WRB
;
3351 pwrb_arr
= kmalloc_array(phba
->params
.cxns_per_ctrl
,
3355 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3356 "BM_%d : Memory alloc failed in create wrb ring.\n");
3359 wrb_vaddr
= mem_descr
->mem_array
[idx
].virtual_address
;
3360 pa_addr_lo
= mem_descr
->mem_array
[idx
].bus_address
.u
.a64
.address
;
3361 num_wrb_rings
= mem_descr
->mem_array
[idx
].size
/
3362 (phba
->params
.wrbs_per_cxn
* sizeof(struct iscsi_wrb
));
3364 for (num
= 0; num
< phba
->params
.cxns_per_ctrl
; num
++) {
3365 if (num_wrb_rings
) {
3366 pwrb_arr
[num
].virtual_address
= wrb_vaddr
;
3367 pwrb_arr
[num
].bus_address
.u
.a64
.address
= pa_addr_lo
;
3368 pwrb_arr
[num
].size
= phba
->params
.wrbs_per_cxn
*
3369 sizeof(struct iscsi_wrb
);
3370 wrb_vaddr
+= pwrb_arr
[num
].size
;
3371 pa_addr_lo
+= pwrb_arr
[num
].size
;
3375 wrb_vaddr
= mem_descr
->mem_array
[idx
].virtual_address
;
3376 pa_addr_lo
= mem_descr
->mem_array
[idx
].\
3377 bus_address
.u
.a64
.address
;
3378 num_wrb_rings
= mem_descr
->mem_array
[idx
].size
/
3379 (phba
->params
.wrbs_per_cxn
*
3380 sizeof(struct iscsi_wrb
));
3381 pwrb_arr
[num
].virtual_address
= wrb_vaddr
;
3382 pwrb_arr
[num
].bus_address
.u
.a64
.address\
3384 pwrb_arr
[num
].size
= phba
->params
.wrbs_per_cxn
*
3385 sizeof(struct iscsi_wrb
);
3386 wrb_vaddr
+= pwrb_arr
[num
].size
;
3387 pa_addr_lo
+= pwrb_arr
[num
].size
;
3392 /* Get the ULP Count */
3393 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++)
3394 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
)) {
3396 ulp_base_num
= ulp_num
;
3397 cid_count_ulp
[ulp_num
] =
3398 BEISCSI_GET_CID_COUNT(phba
, ulp_num
);
3401 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
3402 if (ulp_count
> 1) {
3403 ulp_base_num
= (ulp_base_num
+ 1) % BEISCSI_ULP_COUNT
;
3405 if (!cid_count_ulp
[ulp_base_num
])
3406 ulp_base_num
= (ulp_base_num
+ 1) %
3409 cid_count_ulp
[ulp_base_num
]--;
3413 hwi_build_be_sgl_by_offset(phba
, &pwrb_arr
[i
], &sgl
);
3414 status
= be_cmd_wrbq_create(&phba
->ctrl
, &sgl
,
3415 &phwi_context
->be_wrbq
[i
],
3416 &phwi_ctrlr
->wrb_context
[i
],
3419 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3420 "BM_%d : wrbq create failed.");
3424 pwrb_context
= &phwi_ctrlr
->wrb_context
[i
];
3425 BE_SET_CID_TO_CRI(i
, pwrb_context
->cid
);
3431 static void free_wrb_handles(struct beiscsi_hba
*phba
)
3434 struct hwi_controller
*phwi_ctrlr
;
3435 struct hwi_wrb_context
*pwrb_context
;
3437 phwi_ctrlr
= phba
->phwi_ctrlr
;
3438 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
; index
++) {
3439 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
3440 kfree(pwrb_context
->pwrb_handle_base
);
3441 kfree(pwrb_context
->pwrb_handle_basestd
);
3445 static void be_mcc_queues_destroy(struct beiscsi_hba
*phba
)
3447 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3448 struct be_dma_mem
*ptag_mem
;
3449 struct be_queue_info
*q
;
3452 q
= &phba
->ctrl
.mcc_obj
.q
;
3453 for (i
= 0; i
< MAX_MCC_CMD
; i
++) {
3455 if (!test_bit(MCC_TAG_STATE_RUNNING
,
3456 &ctrl
->ptag_state
[tag
].tag_state
))
3459 if (test_bit(MCC_TAG_STATE_TIMEOUT
,
3460 &ctrl
->ptag_state
[tag
].tag_state
)) {
3461 ptag_mem
= &ctrl
->ptag_state
[tag
].tag_mem_state
;
3462 if (ptag_mem
->size
) {
3463 dma_free_coherent(&ctrl
->pdev
->dev
,
3472 * If MCC is still active and waiting then wake up the process.
3473 * We are here only because port is going offline. The process
3474 * sees that (BEISCSI_HBA_ONLINE is cleared) and EIO error is
3475 * returned for the operation and allocated memory cleaned up.
3477 if (waitqueue_active(&ctrl
->mcc_wait
[tag
])) {
3478 ctrl
->mcc_tag_status
[tag
] = MCC_STATUS_FAILED
;
3479 ctrl
->mcc_tag_status
[tag
] |= CQE_VALID_MASK
;
3480 wake_up_interruptible(&ctrl
->mcc_wait
[tag
]);
3482 * Control tag info gets reinitialized in enable
3483 * so wait for the process to clear running state.
3485 while (test_bit(MCC_TAG_STATE_RUNNING
,
3486 &ctrl
->ptag_state
[tag
].tag_state
))
3487 schedule_timeout_uninterruptible(HZ
);
3490 * For MCC with tag_states MCC_TAG_STATE_ASYNC and
3491 * MCC_TAG_STATE_IGNORE nothing needs to done.
3495 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_MCCQ
);
3496 be_queue_free(phba
, q
);
3499 q
= &phba
->ctrl
.mcc_obj
.cq
;
3501 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_CQ
);
3502 be_queue_free(phba
, q
);
3506 static int be_mcc_queues_create(struct beiscsi_hba
*phba
,
3507 struct hwi_context_memory
*phwi_context
)
3509 struct be_queue_info
*q
, *cq
;
3510 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3512 /* Alloc MCC compl queue */
3513 cq
= &phba
->ctrl
.mcc_obj
.cq
;
3514 if (be_queue_alloc(phba
, cq
, MCC_CQ_LEN
,
3515 sizeof(struct be_mcc_compl
)))
3517 /* Ask BE to create MCC compl queue; */
3518 if (phba
->pcidev
->msix_enabled
) {
3519 if (beiscsi_cmd_cq_create(ctrl
, cq
,
3520 &phwi_context
->be_eq
[phba
->num_cpus
].q
,
3524 if (beiscsi_cmd_cq_create(ctrl
, cq
, &phwi_context
->be_eq
[0].q
,
3529 /* Alloc MCC queue */
3530 q
= &phba
->ctrl
.mcc_obj
.q
;
3531 if (be_queue_alloc(phba
, q
, MCC_Q_LEN
, sizeof(struct be_mcc_wrb
)))
3532 goto mcc_cq_destroy
;
3534 /* Ask BE to create MCC queue */
3535 if (beiscsi_cmd_mccq_create(phba
, q
, cq
))
3541 be_queue_free(phba
, q
);
3543 beiscsi_cmd_q_destroy(ctrl
, cq
, QTYPE_CQ
);
3545 be_queue_free(phba
, cq
);
3550 static void be2iscsi_enable_msix(struct beiscsi_hba
*phba
)
3554 switch (phba
->generation
) {
3557 nvec
= BEISCSI_MAX_NUM_CPUS
+ 1;
3560 nvec
= phba
->fw_config
.eqid_count
;
3567 /* if eqid_count == 1 fall back to INTX */
3568 if (enable_msix
&& nvec
> 1) {
3569 const struct irq_affinity desc
= { .post_vectors
= 1 };
3571 if (pci_alloc_irq_vectors_affinity(phba
->pcidev
, 2, nvec
,
3572 PCI_IRQ_MSIX
| PCI_IRQ_AFFINITY
, &desc
) < 0) {
3573 phba
->num_cpus
= nvec
- 1;
3581 static void hwi_purge_eq(struct beiscsi_hba
*phba
)
3583 struct hwi_controller
*phwi_ctrlr
;
3584 struct hwi_context_memory
*phwi_context
;
3585 struct be_queue_info
*eq
;
3586 struct be_eq_entry
*eqe
= NULL
;
3588 unsigned int num_processed
;
3590 if (beiscsi_hba_in_error(phba
))
3593 phwi_ctrlr
= phba
->phwi_ctrlr
;
3594 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3595 if (phba
->pcidev
->msix_enabled
)
3600 for (i
= 0; i
< (phba
->num_cpus
+ eq_msix
); i
++) {
3601 eq
= &phwi_context
->be_eq
[i
].q
;
3602 eqe
= queue_tail_node(eq
);
3604 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
3606 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
3608 eqe
= queue_tail_node(eq
);
3613 hwi_ring_eq_db(phba
, eq
->id
, 1, num_processed
, 1, 1);
3617 static void hwi_cleanup_port(struct beiscsi_hba
*phba
)
3619 struct be_queue_info
*q
;
3620 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3621 struct hwi_controller
*phwi_ctrlr
;
3622 struct hwi_context_memory
*phwi_context
;
3623 int i
, eq_for_mcc
, ulp_num
;
3625 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++)
3626 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
))
3627 beiscsi_cmd_iscsi_cleanup(phba
, ulp_num
);
3630 * Purge all EQ entries that may have been left out. This is to
3631 * workaround a problem we've seen occasionally where driver gets an
3632 * interrupt with EQ entry bit set after stopping the controller.
3636 phwi_ctrlr
= phba
->phwi_ctrlr
;
3637 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3639 be_cmd_iscsi_remove_template_hdr(ctrl
);
3641 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
3642 q
= &phwi_context
->be_wrbq
[i
];
3644 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_WRBQ
);
3646 kfree(phwi_context
->be_wrbq
);
3647 free_wrb_handles(phba
);
3649 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
3650 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
)) {
3652 q
= &phwi_context
->be_def_hdrq
[ulp_num
];
3654 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_DPDUQ
);
3656 q
= &phwi_context
->be_def_dataq
[ulp_num
];
3658 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_DPDUQ
);
3662 beiscsi_cmd_q_destroy(ctrl
, NULL
, QTYPE_SGL
);
3664 for (i
= 0; i
< (phba
->num_cpus
); i
++) {
3665 q
= &phwi_context
->be_cq
[i
];
3667 be_queue_free(phba
, q
);
3668 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_CQ
);
3672 be_mcc_queues_destroy(phba
);
3673 if (phba
->pcidev
->msix_enabled
)
3677 for (i
= 0; i
< (phba
->num_cpus
+ eq_for_mcc
); i
++) {
3678 q
= &phwi_context
->be_eq
[i
].q
;
3680 be_queue_free(phba
, q
);
3681 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_EQ
);
3684 /* this ensures complete FW cleanup */
3685 beiscsi_cmd_function_reset(phba
);
3686 /* last communication, indicate driver is unloading */
3687 beiscsi_cmd_special_wrb(&phba
->ctrl
, 0);
3690 static int hwi_init_port(struct beiscsi_hba
*phba
)
3692 struct hwi_controller
*phwi_ctrlr
;
3693 struct hwi_context_memory
*phwi_context
;
3694 unsigned int def_pdu_ring_sz
;
3695 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3696 int status
, ulp_num
;
3699 phwi_ctrlr
= phba
->phwi_ctrlr
;
3700 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3701 /* set port optic state to unknown */
3702 phba
->optic_state
= 0xff;
3704 status
= beiscsi_create_eqs(phba
, phwi_context
);
3706 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3707 "BM_%d : EQ not created\n");
3711 status
= be_mcc_queues_create(phba
, phwi_context
);
3715 status
= beiscsi_check_supported_fw(ctrl
, phba
);
3717 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3718 "BM_%d : Unsupported fw version\n");
3722 status
= beiscsi_create_cqs(phba
, phwi_context
);
3724 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3725 "BM_%d : CQ not created\n");
3729 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
3730 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
)) {
3731 nbufs
= phwi_context
->pasync_ctx
[ulp_num
]->num_entries
;
3732 def_pdu_ring_sz
= nbufs
* sizeof(struct phys_addr
);
3734 status
= beiscsi_create_def_hdr(phba
, phwi_context
,
3739 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3740 "BM_%d : Default Header not created for ULP : %d\n",
3745 status
= beiscsi_create_def_data(phba
, phwi_context
,
3750 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3751 "BM_%d : Default Data not created for ULP : %d\n",
3756 * Now that the default PDU rings have been created,
3757 * let EP know about it.
3759 beiscsi_hdq_post_handles(phba
, BEISCSI_DEFQ_HDR
,
3761 beiscsi_hdq_post_handles(phba
, BEISCSI_DEFQ_DATA
,
3766 status
= beiscsi_post_pages(phba
);
3768 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3769 "BM_%d : Post SGL Pages Failed\n");
3773 status
= beiscsi_post_template_hdr(phba
);
3775 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3776 "BM_%d : Template HDR Posting for CXN Failed\n");
3779 status
= beiscsi_create_wrb_rings(phba
, phwi_context
, phwi_ctrlr
);
3781 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3782 "BM_%d : WRB Rings not created\n");
3786 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
3787 uint16_t async_arr_idx
= 0;
3789 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
)) {
3791 struct hd_async_context
*pasync_ctx
;
3793 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(
3794 phwi_ctrlr
, ulp_num
);
3796 phba
->params
.cxns_per_ctrl
; cri
++) {
3797 if (ulp_num
== BEISCSI_GET_ULP_FROM_CRI
3799 pasync_ctx
->cid_to_async_cri_map
[
3800 phwi_ctrlr
->wrb_context
[cri
].cid
] =
3806 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3807 "BM_%d : hwi_init_port success\n");
3811 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3812 "BM_%d : hwi_init_port failed");
3813 hwi_cleanup_port(phba
);
3817 static int hwi_init_controller(struct beiscsi_hba
*phba
)
3819 struct hwi_controller
*phwi_ctrlr
;
3821 phwi_ctrlr
= phba
->phwi_ctrlr
;
3822 if (1 == phba
->init_mem
[HWI_MEM_ADDN_CONTEXT
].num_elements
) {
3823 phwi_ctrlr
->phwi_ctxt
= (struct hwi_context_memory
*)phba
->
3824 init_mem
[HWI_MEM_ADDN_CONTEXT
].mem_array
[0].virtual_address
;
3825 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3826 "BM_%d : phwi_ctrlr->phwi_ctxt=%p\n",
3827 phwi_ctrlr
->phwi_ctxt
);
3829 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3830 "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3831 "than one element.Failing to load\n");
3835 iscsi_init_global_templates(phba
);
3836 if (beiscsi_init_wrb_handle(phba
))
3839 if (hwi_init_async_pdu_ctx(phba
)) {
3840 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3841 "BM_%d : hwi_init_async_pdu_ctx failed\n");
3845 if (hwi_init_port(phba
) != 0) {
3846 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3847 "BM_%d : hwi_init_controller failed\n");
3854 static void beiscsi_free_mem(struct beiscsi_hba
*phba
)
3856 struct be_mem_descriptor
*mem_descr
;
3859 mem_descr
= phba
->init_mem
;
3862 for (i
= 0; i
< SE_MEM_MAX
; i
++) {
3863 for (j
= mem_descr
->num_elements
; j
> 0; j
--) {
3864 dma_free_coherent(&phba
->pcidev
->dev
,
3865 mem_descr
->mem_array
[j
- 1].size
,
3866 mem_descr
->mem_array
[j
- 1].virtual_address
,
3867 (unsigned long)mem_descr
->mem_array
[j
- 1].
3868 bus_address
.u
.a64
.address
);
3871 kfree(mem_descr
->mem_array
);
3874 kfree(phba
->init_mem
);
3875 kfree(phba
->phwi_ctrlr
->wrb_context
);
3876 kfree(phba
->phwi_ctrlr
);
3879 static int beiscsi_init_sgl_handle(struct beiscsi_hba
*phba
)
3881 struct be_mem_descriptor
*mem_descr_sglh
, *mem_descr_sg
;
3882 struct sgl_handle
*psgl_handle
;
3883 struct iscsi_sge
*pfrag
;
3884 unsigned int arr_index
, i
, idx
;
3885 unsigned int ulp_icd_start
, ulp_num
= 0;
3887 phba
->io_sgl_hndl_avbl
= 0;
3888 phba
->eh_sgl_hndl_avbl
= 0;
3890 mem_descr_sglh
= phba
->init_mem
;
3891 mem_descr_sglh
+= HWI_MEM_SGLH
;
3892 if (1 == mem_descr_sglh
->num_elements
) {
3893 phba
->io_sgl_hndl_base
= kcalloc(phba
->params
.ios_per_ctrl
,
3894 sizeof(struct sgl_handle
*),
3896 if (!phba
->io_sgl_hndl_base
) {
3897 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3898 "BM_%d : Mem Alloc Failed. Failing to load\n");
3901 phba
->eh_sgl_hndl_base
=
3902 kcalloc(phba
->params
.icds_per_ctrl
-
3903 phba
->params
.ios_per_ctrl
,
3904 sizeof(struct sgl_handle
*), GFP_KERNEL
);
3905 if (!phba
->eh_sgl_hndl_base
) {
3906 kfree(phba
->io_sgl_hndl_base
);
3907 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3908 "BM_%d : Mem Alloc Failed. Failing to load\n");
3912 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
3913 "BM_%d : HWI_MEM_SGLH is more than one element."
3914 "Failing to load\n");
3920 while (idx
< mem_descr_sglh
->num_elements
) {
3921 psgl_handle
= mem_descr_sglh
->mem_array
[idx
].virtual_address
;
3923 for (i
= 0; i
< (mem_descr_sglh
->mem_array
[idx
].size
/
3924 sizeof(struct sgl_handle
)); i
++) {
3925 if (arr_index
< phba
->params
.ios_per_ctrl
) {
3926 phba
->io_sgl_hndl_base
[arr_index
] = psgl_handle
;
3927 phba
->io_sgl_hndl_avbl
++;
3930 phba
->eh_sgl_hndl_base
[arr_index
-
3931 phba
->params
.ios_per_ctrl
] =
3934 phba
->eh_sgl_hndl_avbl
++;
3940 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3941 "BM_%d : phba->io_sgl_hndl_avbl=%d"
3942 "phba->eh_sgl_hndl_avbl=%d\n",
3943 phba
->io_sgl_hndl_avbl
,
3944 phba
->eh_sgl_hndl_avbl
);
3946 mem_descr_sg
= phba
->init_mem
;
3947 mem_descr_sg
+= HWI_MEM_SGE
;
3948 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
3949 "\n BM_%d : mem_descr_sg->num_elements=%d\n",
3950 mem_descr_sg
->num_elements
);
3952 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++)
3953 if (test_bit(ulp_num
, &phba
->fw_config
.ulp_supported
))
3956 ulp_icd_start
= phba
->fw_config
.iscsi_icd_start
[ulp_num
];
3960 while (idx
< mem_descr_sg
->num_elements
) {
3961 pfrag
= mem_descr_sg
->mem_array
[idx
].virtual_address
;
3964 i
< (mem_descr_sg
->mem_array
[idx
].size
) /
3965 (sizeof(struct iscsi_sge
) * phba
->params
.num_sge_per_io
);
3967 if (arr_index
< phba
->params
.ios_per_ctrl
)
3968 psgl_handle
= phba
->io_sgl_hndl_base
[arr_index
];
3970 psgl_handle
= phba
->eh_sgl_hndl_base
[arr_index
-
3971 phba
->params
.ios_per_ctrl
];
3972 psgl_handle
->pfrag
= pfrag
;
3973 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, pfrag
, 0);
3974 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, pfrag
, 0);
3975 pfrag
+= phba
->params
.num_sge_per_io
;
3976 psgl_handle
->sgl_index
= ulp_icd_start
+ arr_index
++;
3980 phba
->io_sgl_free_index
= 0;
3981 phba
->io_sgl_alloc_index
= 0;
3982 phba
->eh_sgl_free_index
= 0;
3983 phba
->eh_sgl_alloc_index
= 0;
3987 static int hba_setup_cid_tbls(struct beiscsi_hba
*phba
)
3990 uint16_t i
, ulp_num
;
3991 struct ulp_cid_info
*ptr_cid_info
= NULL
;
3993 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
3994 if (test_bit(ulp_num
, (void *)&phba
->fw_config
.ulp_supported
)) {
3995 ptr_cid_info
= kzalloc(sizeof(struct ulp_cid_info
),
3998 if (!ptr_cid_info
) {
3999 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
4000 "BM_%d : Failed to allocate memory"
4001 "for ULP_CID_INFO for ULP : %d\n",
4008 /* Allocate memory for CID array */
4009 ptr_cid_info
->cid_array
=
4010 kcalloc(BEISCSI_GET_CID_COUNT(phba
, ulp_num
),
4011 sizeof(*ptr_cid_info
->cid_array
),
4013 if (!ptr_cid_info
->cid_array
) {
4014 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
4015 "BM_%d : Failed to allocate memory"
4016 "for CID_ARRAY for ULP : %d\n",
4018 kfree(ptr_cid_info
);
4019 ptr_cid_info
= NULL
;
4024 ptr_cid_info
->avlbl_cids
= BEISCSI_GET_CID_COUNT(
4027 /* Save the cid_info_array ptr */
4028 phba
->cid_array_info
[ulp_num
] = ptr_cid_info
;
4031 phba
->ep_array
= kcalloc(phba
->params
.cxns_per_ctrl
,
4032 sizeof(struct iscsi_endpoint
*),
4034 if (!phba
->ep_array
) {
4035 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
4036 "BM_%d : Failed to allocate memory in "
4037 "hba_setup_cid_tbls\n");
4043 phba
->conn_table
= kcalloc(phba
->params
.cxns_per_ctrl
,
4044 sizeof(struct beiscsi_conn
*),
4046 if (!phba
->conn_table
) {
4047 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
4048 "BM_%d : Failed to allocate memory in"
4049 "hba_setup_cid_tbls\n");
4051 kfree(phba
->ep_array
);
4052 phba
->ep_array
= NULL
;
4058 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
4059 ulp_num
= phba
->phwi_ctrlr
->wrb_context
[i
].ulp_num
;
4061 ptr_cid_info
= phba
->cid_array_info
[ulp_num
];
4062 ptr_cid_info
->cid_array
[ptr_cid_info
->cid_alloc
++] =
4063 phba
->phwi_ctrlr
->wrb_context
[i
].cid
;
4067 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
4068 if (test_bit(ulp_num
, (void *)&phba
->fw_config
.ulp_supported
)) {
4069 ptr_cid_info
= phba
->cid_array_info
[ulp_num
];
4071 ptr_cid_info
->cid_alloc
= 0;
4072 ptr_cid_info
->cid_free
= 0;
4078 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
4079 if (test_bit(ulp_num
, (void *)&phba
->fw_config
.ulp_supported
)) {
4080 ptr_cid_info
= phba
->cid_array_info
[ulp_num
];
4083 kfree(ptr_cid_info
->cid_array
);
4084 kfree(ptr_cid_info
);
4085 phba
->cid_array_info
[ulp_num
] = NULL
;
4093 static void hwi_enable_intr(struct beiscsi_hba
*phba
)
4095 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
4096 struct hwi_controller
*phwi_ctrlr
;
4097 struct hwi_context_memory
*phwi_context
;
4098 struct be_queue_info
*eq
;
4103 phwi_ctrlr
= phba
->phwi_ctrlr
;
4104 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
4106 addr
= (u8 __iomem
*) ((u8 __iomem
*) ctrl
->pcicfg
+
4107 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET
);
4108 reg
= ioread32(addr
);
4110 enabled
= reg
& MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
4112 reg
|= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
4113 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
4114 "BM_%d : reg =x%08x addr=%p\n", reg
, addr
);
4115 iowrite32(reg
, addr
);
4118 if (!phba
->pcidev
->msix_enabled
) {
4119 eq
= &phwi_context
->be_eq
[0].q
;
4120 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
4121 "BM_%d : eq->id=%d\n", eq
->id
);
4123 hwi_ring_eq_db(phba
, eq
->id
, 0, 0, 1, 1);
4125 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
4126 eq
= &phwi_context
->be_eq
[i
].q
;
4127 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
4128 "BM_%d : eq->id=%d\n", eq
->id
);
4129 hwi_ring_eq_db(phba
, eq
->id
, 0, 0, 1, 1);
4134 static void hwi_disable_intr(struct beiscsi_hba
*phba
)
4136 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
4138 u8 __iomem
*addr
= ctrl
->pcicfg
+ PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET
;
4139 u32 reg
= ioread32(addr
);
4141 u32 enabled
= reg
& MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
4143 reg
&= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
4144 iowrite32(reg
, addr
);
4146 beiscsi_log(phba
, KERN_WARNING
, BEISCSI_LOG_INIT
,
4147 "BM_%d : In hwi_disable_intr, Already Disabled\n");
4150 static int beiscsi_init_port(struct beiscsi_hba
*phba
)
4154 ret
= hwi_init_controller(phba
);
4156 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
4157 "BM_%d : init controller failed\n");
4160 ret
= beiscsi_init_sgl_handle(phba
);
4162 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
4163 "BM_%d : init sgl handles failed\n");
4167 ret
= hba_setup_cid_tbls(phba
);
4169 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
4170 "BM_%d : setup CID table failed\n");
4171 kfree(phba
->io_sgl_hndl_base
);
4172 kfree(phba
->eh_sgl_hndl_base
);
4178 hwi_cleanup_port(phba
);
4182 static void beiscsi_cleanup_port(struct beiscsi_hba
*phba
)
4184 struct ulp_cid_info
*ptr_cid_info
= NULL
;
4187 kfree(phba
->io_sgl_hndl_base
);
4188 kfree(phba
->eh_sgl_hndl_base
);
4189 kfree(phba
->ep_array
);
4190 kfree(phba
->conn_table
);
4192 for (ulp_num
= 0; ulp_num
< BEISCSI_ULP_COUNT
; ulp_num
++) {
4193 if (test_bit(ulp_num
, (void *)&phba
->fw_config
.ulp_supported
)) {
4194 ptr_cid_info
= phba
->cid_array_info
[ulp_num
];
4197 kfree(ptr_cid_info
->cid_array
);
4198 kfree(ptr_cid_info
);
4199 phba
->cid_array_info
[ulp_num
] = NULL
;
4206 * beiscsi_free_mgmt_task_handles()- Free driver CXN resources
4207 * @beiscsi_conn: ptr to the conn to be cleaned up
4208 * @task: ptr to iscsi_task resource to be freed.
4210 * Free driver mgmt resources binded to CXN.
4213 beiscsi_free_mgmt_task_handles(struct beiscsi_conn
*beiscsi_conn
,
4214 struct iscsi_task
*task
)
4216 struct beiscsi_io_task
*io_task
;
4217 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
4218 struct hwi_wrb_context
*pwrb_context
;
4219 struct hwi_controller
*phwi_ctrlr
;
4220 uint16_t cri_index
= BE_GET_CRI_FROM_CID(
4221 beiscsi_conn
->beiscsi_conn_cid
);
4223 phwi_ctrlr
= phba
->phwi_ctrlr
;
4224 pwrb_context
= &phwi_ctrlr
->wrb_context
[cri_index
];
4226 io_task
= task
->dd_data
;
4228 if (io_task
->pwrb_handle
) {
4229 free_wrb_handle(phba
, pwrb_context
, io_task
->pwrb_handle
);
4230 io_task
->pwrb_handle
= NULL
;
4233 if (io_task
->psgl_handle
) {
4234 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
4235 io_task
->psgl_handle
= NULL
;
4238 if (io_task
->mtask_addr
) {
4239 dma_unmap_single(&phba
->pcidev
->dev
,
4240 io_task
->mtask_addr
,
4241 io_task
->mtask_data_count
,
4243 io_task
->mtask_addr
= 0;
4248 * beiscsi_cleanup_task()- Free driver resources of the task
4249 * @task: ptr to the iscsi task
4252 static void beiscsi_cleanup_task(struct iscsi_task
*task
)
4254 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4255 struct iscsi_conn
*conn
= task
->conn
;
4256 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
4257 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
4258 struct beiscsi_session
*beiscsi_sess
= beiscsi_conn
->beiscsi_sess
;
4259 struct hwi_wrb_context
*pwrb_context
;
4260 struct hwi_controller
*phwi_ctrlr
;
4261 uint16_t cri_index
= BE_GET_CRI_FROM_CID(
4262 beiscsi_conn
->beiscsi_conn_cid
);
4264 phwi_ctrlr
= phba
->phwi_ctrlr
;
4265 pwrb_context
= &phwi_ctrlr
->wrb_context
[cri_index
];
4267 if (io_task
->cmd_bhs
) {
4268 dma_pool_free(beiscsi_sess
->bhs_pool
, io_task
->cmd_bhs
,
4269 io_task
->bhs_pa
.u
.a64
.address
);
4270 io_task
->cmd_bhs
= NULL
;
4275 if (io_task
->pwrb_handle
) {
4276 free_wrb_handle(phba
, pwrb_context
,
4277 io_task
->pwrb_handle
);
4278 io_task
->pwrb_handle
= NULL
;
4281 if (io_task
->psgl_handle
) {
4282 free_io_sgl_handle(phba
, io_task
->psgl_handle
);
4283 io_task
->psgl_handle
= NULL
;
4286 if (io_task
->scsi_cmnd
) {
4287 if (io_task
->num_sg
)
4288 scsi_dma_unmap(io_task
->scsi_cmnd
);
4289 io_task
->scsi_cmnd
= NULL
;
4292 if (!beiscsi_conn
->login_in_progress
)
4293 beiscsi_free_mgmt_task_handles(beiscsi_conn
, task
);
4298 beiscsi_offload_connection(struct beiscsi_conn
*beiscsi_conn
,
4299 struct beiscsi_offload_params
*params
)
4301 struct wrb_handle
*pwrb_handle
;
4302 struct hwi_wrb_context
*pwrb_context
= NULL
;
4303 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
4304 struct iscsi_task
*task
= beiscsi_conn
->task
;
4305 struct iscsi_session
*session
= task
->conn
->session
;
4309 * We can always use 0 here because it is reserved by libiscsi for
4310 * login/startup related tasks.
4312 beiscsi_conn
->login_in_progress
= 0;
4313 spin_lock_bh(&session
->back_lock
);
4314 beiscsi_cleanup_task(task
);
4315 spin_unlock_bh(&session
->back_lock
);
4317 pwrb_handle
= alloc_wrb_handle(phba
, beiscsi_conn
->beiscsi_conn_cid
,
4320 /* Check for the adapter family */
4321 if (is_chip_be2_be3r(phba
))
4322 beiscsi_offload_cxn_v0(params
, pwrb_handle
,
4326 beiscsi_offload_cxn_v2(params
, pwrb_handle
,
4329 be_dws_le_to_cpu(pwrb_handle
->pwrb
,
4330 sizeof(struct iscsi_target_context_update_wrb
));
4332 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
4333 doorbell
|= (pwrb_handle
->wrb_index
& DB_DEF_PDU_WRB_INDEX_MASK
)
4334 << DB_DEF_PDU_WRB_INDEX_SHIFT
;
4335 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
4336 iowrite32(doorbell
, phba
->db_va
+
4337 beiscsi_conn
->doorbell_offset
);
4340 * There is no completion for CONTEXT_UPDATE. The completion of next
4341 * WRB posted guarantees FW's processing and DMA'ing of it.
4342 * Use beiscsi_put_wrb_handle to put it back in the pool which makes
4343 * sure zero'ing or reuse of the WRB only after wrbs_per_cxn.
4345 beiscsi_put_wrb_handle(pwrb_context
, pwrb_handle
,
4346 phba
->params
.wrbs_per_cxn
);
4347 beiscsi_log(phba
, KERN_INFO
,
4348 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
4349 "BM_%d : put CONTEXT_UPDATE pwrb_handle=%p free_index=0x%x wrb_handles_available=%d\n",
4350 pwrb_handle
, pwrb_context
->free_index
,
4351 pwrb_context
->wrb_handles_available
);
4354 static void beiscsi_parse_pdu(struct iscsi_conn
*conn
, itt_t itt
,
4355 int *index
, int *age
)
4359 *age
= conn
->session
->age
;
4363 * beiscsi_alloc_pdu - allocates pdu and related resources
4364 * @task: libiscsi task
4365 * @opcode: opcode of pdu for task
4367 * This is called with the session lock held. It will allocate
4368 * the wrb and sgl if needed for the command. And it will prep
4369 * the pdu's itt. beiscsi_parse_pdu will later translate
4370 * the pdu itt to the libiscsi task itt.
4372 static int beiscsi_alloc_pdu(struct iscsi_task
*task
, uint8_t opcode
)
4374 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4375 struct iscsi_conn
*conn
= task
->conn
;
4376 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
4377 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
4378 struct hwi_wrb_context
*pwrb_context
;
4379 struct hwi_controller
*phwi_ctrlr
;
4381 uint16_t cri_index
= 0;
4382 struct beiscsi_session
*beiscsi_sess
= beiscsi_conn
->beiscsi_sess
;
4385 io_task
->cmd_bhs
= dma_pool_alloc(beiscsi_sess
->bhs_pool
,
4386 GFP_ATOMIC
, &paddr
);
4387 if (!io_task
->cmd_bhs
)
4389 io_task
->bhs_pa
.u
.a64
.address
= paddr
;
4390 io_task
->libiscsi_itt
= (itt_t
)task
->itt
;
4391 io_task
->conn
= beiscsi_conn
;
4393 task
->hdr
= (struct iscsi_hdr
*)&io_task
->cmd_bhs
->iscsi_hdr
;
4394 task
->hdr_max
= sizeof(struct be_cmd_bhs
);
4395 io_task
->psgl_handle
= NULL
;
4396 io_task
->pwrb_handle
= NULL
;
4399 io_task
->psgl_handle
= alloc_io_sgl_handle(phba
);
4400 if (!io_task
->psgl_handle
) {
4401 beiscsi_log(phba
, KERN_ERR
,
4402 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
4403 "BM_%d : Alloc of IO_SGL_ICD Failed"
4404 "for the CID : %d\n",
4405 beiscsi_conn
->beiscsi_conn_cid
);
4408 io_task
->pwrb_handle
= alloc_wrb_handle(phba
,
4409 beiscsi_conn
->beiscsi_conn_cid
,
4410 &io_task
->pwrb_context
);
4411 if (!io_task
->pwrb_handle
) {
4412 beiscsi_log(phba
, KERN_ERR
,
4413 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
4414 "BM_%d : Alloc of WRB_HANDLE Failed"
4415 "for the CID : %d\n",
4416 beiscsi_conn
->beiscsi_conn_cid
);
4420 io_task
->scsi_cmnd
= NULL
;
4421 if ((opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGIN
) {
4422 beiscsi_conn
->task
= task
;
4423 if (!beiscsi_conn
->login_in_progress
) {
4424 io_task
->psgl_handle
= (struct sgl_handle
*)
4425 alloc_mgmt_sgl_handle(phba
);
4426 if (!io_task
->psgl_handle
) {
4427 beiscsi_log(phba
, KERN_ERR
,
4430 "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4431 "for the CID : %d\n",
4437 beiscsi_conn
->login_in_progress
= 1;
4438 beiscsi_conn
->plogin_sgl_handle
=
4439 io_task
->psgl_handle
;
4440 io_task
->pwrb_handle
=
4441 alloc_wrb_handle(phba
,
4442 beiscsi_conn
->beiscsi_conn_cid
,
4443 &io_task
->pwrb_context
);
4444 if (!io_task
->pwrb_handle
) {
4445 beiscsi_log(phba
, KERN_ERR
,
4448 "BM_%d : Alloc of WRB_HANDLE Failed"
4449 "for the CID : %d\n",
4452 goto free_mgmt_hndls
;
4454 beiscsi_conn
->plogin_wrb_handle
=
4455 io_task
->pwrb_handle
;
4458 io_task
->psgl_handle
=
4459 beiscsi_conn
->plogin_sgl_handle
;
4460 io_task
->pwrb_handle
=
4461 beiscsi_conn
->plogin_wrb_handle
;
4464 io_task
->psgl_handle
= alloc_mgmt_sgl_handle(phba
);
4465 if (!io_task
->psgl_handle
) {
4466 beiscsi_log(phba
, KERN_ERR
,
4469 "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4470 "for the CID : %d\n",
4475 io_task
->pwrb_handle
=
4476 alloc_wrb_handle(phba
,
4477 beiscsi_conn
->beiscsi_conn_cid
,
4478 &io_task
->pwrb_context
);
4479 if (!io_task
->pwrb_handle
) {
4480 beiscsi_log(phba
, KERN_ERR
,
4481 BEISCSI_LOG_IO
| BEISCSI_LOG_CONFIG
,
4482 "BM_%d : Alloc of WRB_HANDLE Failed"
4483 "for the CID : %d\n",
4484 beiscsi_conn
->beiscsi_conn_cid
);
4485 goto free_mgmt_hndls
;
4490 itt
= (itt_t
) cpu_to_be32(((unsigned int)io_task
->pwrb_handle
->
4491 wrb_index
<< 16) | (unsigned int)
4492 (io_task
->psgl_handle
->sgl_index
));
4493 io_task
->pwrb_handle
->pio_handle
= task
;
4495 io_task
->cmd_bhs
->iscsi_hdr
.itt
= itt
;
4499 free_io_sgl_handle(phba
, io_task
->psgl_handle
);
4502 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
4503 io_task
->psgl_handle
= NULL
;
4505 phwi_ctrlr
= phba
->phwi_ctrlr
;
4506 cri_index
= BE_GET_CRI_FROM_CID(
4507 beiscsi_conn
->beiscsi_conn_cid
);
4508 pwrb_context
= &phwi_ctrlr
->wrb_context
[cri_index
];
4509 if (io_task
->pwrb_handle
)
4510 free_wrb_handle(phba
, pwrb_context
, io_task
->pwrb_handle
);
4511 io_task
->pwrb_handle
= NULL
;
4512 dma_pool_free(beiscsi_sess
->bhs_pool
, io_task
->cmd_bhs
,
4513 io_task
->bhs_pa
.u
.a64
.address
);
4514 io_task
->cmd_bhs
= NULL
;
4517 static int beiscsi_iotask_v2(struct iscsi_task
*task
, struct scatterlist
*sg
,
4518 unsigned int num_sg
, unsigned int xferlen
,
4519 unsigned int writedir
)
4522 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4523 struct iscsi_conn
*conn
= task
->conn
;
4524 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
4525 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
4526 struct iscsi_wrb
*pwrb
= NULL
;
4527 unsigned int doorbell
= 0;
4529 pwrb
= io_task
->pwrb_handle
->pwrb
;
4531 io_task
->bhs_len
= sizeof(struct be_cmd_bhs
);
4534 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, type
, pwrb
,
4536 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, dsp
, pwrb
, 1);
4538 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, type
, pwrb
,
4540 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, dsp
, pwrb
, 0);
4543 io_task
->wrb_type
= AMAP_GET_BITS(struct amap_iscsi_wrb_v2
,
4546 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, lun
, pwrb
,
4547 cpu_to_be16(*(unsigned short *)
4548 &io_task
->cmd_bhs
->iscsi_hdr
.lun
));
4549 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, r2t_exp_dtl
, pwrb
, xferlen
);
4550 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, wrb_idx
, pwrb
,
4551 io_task
->pwrb_handle
->wrb_index
);
4552 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, cmdsn_itt
, pwrb
,
4553 be32_to_cpu(task
->cmdsn
));
4554 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sgl_idx
, pwrb
,
4555 io_task
->psgl_handle
->sgl_index
);
4557 hwi_write_sgl_v2(pwrb
, sg
, num_sg
, io_task
);
4558 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, ptr2nextwrb
, pwrb
,
4559 io_task
->pwrb_handle
->wrb_index
);
4560 if (io_task
->pwrb_context
->plast_wrb
)
4561 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, ptr2nextwrb
,
4562 io_task
->pwrb_context
->plast_wrb
,
4563 io_task
->pwrb_handle
->wrb_index
);
4564 io_task
->pwrb_context
->plast_wrb
= pwrb
;
4566 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_wrb
));
4568 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
4569 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
4570 DB_DEF_PDU_WRB_INDEX_MASK
) <<
4571 DB_DEF_PDU_WRB_INDEX_SHIFT
;
4572 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
4573 iowrite32(doorbell
, phba
->db_va
+
4574 beiscsi_conn
->doorbell_offset
);
4578 static int beiscsi_iotask(struct iscsi_task
*task
, struct scatterlist
*sg
,
4579 unsigned int num_sg
, unsigned int xferlen
,
4580 unsigned int writedir
)
4583 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4584 struct iscsi_conn
*conn
= task
->conn
;
4585 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
4586 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
4587 struct iscsi_wrb
*pwrb
= NULL
;
4588 unsigned int doorbell
= 0;
4590 pwrb
= io_task
->pwrb_handle
->pwrb
;
4591 io_task
->bhs_len
= sizeof(struct be_cmd_bhs
);
4594 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4596 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 1);
4598 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4600 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
4603 io_task
->wrb_type
= AMAP_GET_BITS(struct amap_iscsi_wrb
,
4606 AMAP_SET_BITS(struct amap_iscsi_wrb
, lun
, pwrb
,
4607 cpu_to_be16(*(unsigned short *)
4608 &io_task
->cmd_bhs
->iscsi_hdr
.lun
));
4609 AMAP_SET_BITS(struct amap_iscsi_wrb
, r2t_exp_dtl
, pwrb
, xferlen
);
4610 AMAP_SET_BITS(struct amap_iscsi_wrb
, wrb_idx
, pwrb
,
4611 io_task
->pwrb_handle
->wrb_index
);
4612 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
,
4613 be32_to_cpu(task
->cmdsn
));
4614 AMAP_SET_BITS(struct amap_iscsi_wrb
, sgl_icd_idx
, pwrb
,
4615 io_task
->psgl_handle
->sgl_index
);
4617 hwi_write_sgl(pwrb
, sg
, num_sg
, io_task
);
4619 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
, pwrb
,
4620 io_task
->pwrb_handle
->wrb_index
);
4621 if (io_task
->pwrb_context
->plast_wrb
)
4622 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
,
4623 io_task
->pwrb_context
->plast_wrb
,
4624 io_task
->pwrb_handle
->wrb_index
);
4625 io_task
->pwrb_context
->plast_wrb
= pwrb
;
4627 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_wrb
));
4629 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
4630 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
4631 DB_DEF_PDU_WRB_INDEX_MASK
) << DB_DEF_PDU_WRB_INDEX_SHIFT
;
4632 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
4634 iowrite32(doorbell
, phba
->db_va
+
4635 beiscsi_conn
->doorbell_offset
);
4639 static int beiscsi_mtask(struct iscsi_task
*task
)
4641 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4642 struct iscsi_conn
*conn
= task
->conn
;
4643 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
4644 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
4645 struct iscsi_wrb
*pwrb
= NULL
;
4646 unsigned int doorbell
= 0;
4648 unsigned int pwrb_typeoffset
= 0;
4651 cid
= beiscsi_conn
->beiscsi_conn_cid
;
4652 pwrb
= io_task
->pwrb_handle
->pwrb
;
4654 if (is_chip_be2_be3r(phba
)) {
4655 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
,
4656 be32_to_cpu(task
->cmdsn
));
4657 AMAP_SET_BITS(struct amap_iscsi_wrb
, wrb_idx
, pwrb
,
4658 io_task
->pwrb_handle
->wrb_index
);
4659 AMAP_SET_BITS(struct amap_iscsi_wrb
, sgl_icd_idx
, pwrb
,
4660 io_task
->psgl_handle
->sgl_index
);
4661 AMAP_SET_BITS(struct amap_iscsi_wrb
, r2t_exp_dtl
, pwrb
,
4663 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
, pwrb
,
4664 io_task
->pwrb_handle
->wrb_index
);
4665 if (io_task
->pwrb_context
->plast_wrb
)
4666 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
,
4667 io_task
->pwrb_context
->plast_wrb
,
4668 io_task
->pwrb_handle
->wrb_index
);
4669 io_task
->pwrb_context
->plast_wrb
= pwrb
;
4671 pwrb_typeoffset
= BE_WRB_TYPE_OFFSET
;
4673 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, cmdsn_itt
, pwrb
,
4674 be32_to_cpu(task
->cmdsn
));
4675 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, wrb_idx
, pwrb
,
4676 io_task
->pwrb_handle
->wrb_index
);
4677 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, sgl_idx
, pwrb
,
4678 io_task
->psgl_handle
->sgl_index
);
4679 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, r2t_exp_dtl
, pwrb
,
4681 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, ptr2nextwrb
, pwrb
,
4682 io_task
->pwrb_handle
->wrb_index
);
4683 if (io_task
->pwrb_context
->plast_wrb
)
4684 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
, ptr2nextwrb
,
4685 io_task
->pwrb_context
->plast_wrb
,
4686 io_task
->pwrb_handle
->wrb_index
);
4687 io_task
->pwrb_context
->plast_wrb
= pwrb
;
4689 pwrb_typeoffset
= SKH_WRB_TYPE_OFFSET
;
4693 switch (task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) {
4694 case ISCSI_OP_LOGIN
:
4695 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
, 1);
4696 ADAPTER_SET_WRB_TYPE(pwrb
, TGT_DM_CMD
, pwrb_typeoffset
);
4697 ret
= hwi_write_buffer(pwrb
, task
);
4699 case ISCSI_OP_NOOP_OUT
:
4700 if (task
->hdr
->ttt
!= ISCSI_RESERVED_TAG
) {
4701 ADAPTER_SET_WRB_TYPE(pwrb
, TGT_DM_CMD
, pwrb_typeoffset
);
4702 if (is_chip_be2_be3r(phba
))
4703 AMAP_SET_BITS(struct amap_iscsi_wrb
,
4706 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
,
4709 ADAPTER_SET_WRB_TYPE(pwrb
, INI_RD_CMD
, pwrb_typeoffset
);
4710 if (is_chip_be2_be3r(phba
))
4711 AMAP_SET_BITS(struct amap_iscsi_wrb
,
4714 AMAP_SET_BITS(struct amap_iscsi_wrb_v2
,
4717 ret
= hwi_write_buffer(pwrb
, task
);
4720 ADAPTER_SET_WRB_TYPE(pwrb
, TGT_DM_CMD
, pwrb_typeoffset
);
4721 ret
= hwi_write_buffer(pwrb
, task
);
4723 case ISCSI_OP_SCSI_TMFUNC
:
4724 ADAPTER_SET_WRB_TYPE(pwrb
, INI_TMF_CMD
, pwrb_typeoffset
);
4725 ret
= hwi_write_buffer(pwrb
, task
);
4727 case ISCSI_OP_LOGOUT
:
4728 ADAPTER_SET_WRB_TYPE(pwrb
, HWH_TYPE_LOGOUT
, pwrb_typeoffset
);
4729 ret
= hwi_write_buffer(pwrb
, task
);
4733 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_CONFIG
,
4734 "BM_%d : opcode =%d Not supported\n",
4735 task
->hdr
->opcode
& ISCSI_OPCODE_MASK
);
4743 /* Set the task type */
4744 io_task
->wrb_type
= (is_chip_be2_be3r(phba
)) ?
4745 AMAP_GET_BITS(struct amap_iscsi_wrb
, type
, pwrb
) :
4746 AMAP_GET_BITS(struct amap_iscsi_wrb_v2
, type
, pwrb
);
4748 doorbell
|= cid
& DB_WRB_POST_CID_MASK
;
4749 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
4750 DB_DEF_PDU_WRB_INDEX_MASK
) << DB_DEF_PDU_WRB_INDEX_SHIFT
;
4751 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
4752 iowrite32(doorbell
, phba
->db_va
+
4753 beiscsi_conn
->doorbell_offset
);
4757 static int beiscsi_task_xmit(struct iscsi_task
*task
)
4759 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4760 struct scsi_cmnd
*sc
= task
->sc
;
4761 struct beiscsi_hba
*phba
;
4762 struct scatterlist
*sg
;
4764 unsigned int writedir
= 0, xferlen
= 0;
4766 phba
= io_task
->conn
->phba
;
4768 * HBA in error includes BEISCSI_HBA_FW_TIMEOUT. IO path might be
4769 * operational if FW still gets heartbeat from EP FW. Is management
4770 * path really needed to continue further?
4772 if (!beiscsi_hba_is_online(phba
))
4775 if (!io_task
->conn
->login_in_progress
)
4776 task
->hdr
->exp_statsn
= 0;
4779 return beiscsi_mtask(task
);
4781 io_task
->scsi_cmnd
= sc
;
4782 io_task
->num_sg
= 0;
4783 num_sg
= scsi_dma_map(sc
);
4785 beiscsi_log(phba
, KERN_ERR
,
4786 BEISCSI_LOG_IO
| BEISCSI_LOG_ISCSI
,
4787 "BM_%d : scsi_dma_map Failed "
4788 "Driver_ITT : 0x%x ITT : 0x%x Xferlen : 0x%x\n",
4789 be32_to_cpu(io_task
->cmd_bhs
->iscsi_hdr
.itt
),
4790 io_task
->libiscsi_itt
, scsi_bufflen(sc
));
4795 * For scsi cmd task, check num_sg before unmapping in cleanup_task.
4796 * For management task, cleanup_task checks mtask_addr before unmapping.
4798 io_task
->num_sg
= num_sg
;
4799 xferlen
= scsi_bufflen(sc
);
4800 sg
= scsi_sglist(sc
);
4801 if (sc
->sc_data_direction
== DMA_TO_DEVICE
)
4806 return phba
->iotask_fn(task
, sg
, num_sg
, xferlen
, writedir
);
4810 * beiscsi_bsg_request - handle bsg request from ISCSI transport
4811 * @job: job to handle
4813 static int beiscsi_bsg_request(struct bsg_job
*job
)
4815 struct Scsi_Host
*shost
;
4816 struct beiscsi_hba
*phba
;
4817 struct iscsi_bsg_request
*bsg_req
= job
->request
;
4820 struct be_dma_mem nonemb_cmd
;
4821 struct be_cmd_resp_hdr
*resp
;
4822 struct iscsi_bsg_reply
*bsg_reply
= job
->reply
;
4823 unsigned short status
, extd_status
;
4825 shost
= iscsi_job_to_shost(job
);
4826 phba
= iscsi_host_priv(shost
);
4828 if (!beiscsi_hba_is_online(phba
)) {
4829 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_CONFIG
,
4830 "BM_%d : HBA in error 0x%lx\n", phba
->state
);
4834 switch (bsg_req
->msgcode
) {
4835 case ISCSI_BSG_HST_VENDOR
:
4836 nonemb_cmd
.va
= dma_alloc_coherent(&phba
->ctrl
.pdev
->dev
,
4837 job
->request_payload
.payload_len
,
4838 &nonemb_cmd
.dma
, GFP_KERNEL
);
4839 if (nonemb_cmd
.va
== NULL
) {
4840 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_CONFIG
,
4841 "BM_%d : Failed to allocate memory for "
4842 "beiscsi_bsg_request\n");
4845 tag
= mgmt_vendor_specific_fw_cmd(&phba
->ctrl
, phba
, job
,
4848 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_CONFIG
,
4849 "BM_%d : MBX Tag Allocation Failed\n");
4851 dma_free_coherent(&phba
->ctrl
.pdev
->dev
, nonemb_cmd
.size
,
4852 nonemb_cmd
.va
, nonemb_cmd
.dma
);
4856 rc
= wait_event_interruptible_timeout(
4857 phba
->ctrl
.mcc_wait
[tag
],
4858 phba
->ctrl
.mcc_tag_status
[tag
],
4860 BEISCSI_HOST_MBX_TIMEOUT
));
4862 if (!test_bit(BEISCSI_HBA_ONLINE
, &phba
->state
)) {
4863 clear_bit(MCC_TAG_STATE_RUNNING
,
4864 &phba
->ctrl
.ptag_state
[tag
].tag_state
);
4865 dma_free_coherent(&phba
->ctrl
.pdev
->dev
, nonemb_cmd
.size
,
4866 nonemb_cmd
.va
, nonemb_cmd
.dma
);
4869 extd_status
= (phba
->ctrl
.mcc_tag_status
[tag
] &
4870 CQE_STATUS_ADDL_MASK
) >> CQE_STATUS_ADDL_SHIFT
;
4871 status
= phba
->ctrl
.mcc_tag_status
[tag
] & CQE_STATUS_MASK
;
4872 free_mcc_wrb(&phba
->ctrl
, tag
);
4873 resp
= (struct be_cmd_resp_hdr
*)nonemb_cmd
.va
;
4874 sg_copy_from_buffer(job
->reply_payload
.sg_list
,
4875 job
->reply_payload
.sg_cnt
,
4876 nonemb_cmd
.va
, (resp
->response_length
4878 bsg_reply
->reply_payload_rcv_len
= resp
->response_length
;
4879 bsg_reply
->result
= status
;
4880 bsg_job_done(job
, bsg_reply
->result
,
4881 bsg_reply
->reply_payload_rcv_len
);
4882 dma_free_coherent(&phba
->ctrl
.pdev
->dev
, nonemb_cmd
.size
,
4883 nonemb_cmd
.va
, nonemb_cmd
.dma
);
4884 if (status
|| extd_status
) {
4885 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_CONFIG
,
4886 "BM_%d : MBX Cmd Failed"
4887 " status = %d extd_status = %d\n",
4888 status
, extd_status
);
4897 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_CONFIG
,
4898 "BM_%d : Unsupported bsg command: 0x%x\n",
4906 static void beiscsi_hba_attrs_init(struct beiscsi_hba
*phba
)
4908 /* Set the logging parameter */
4909 beiscsi_log_enable_init(phba
, beiscsi_log_enable
);
4912 void beiscsi_start_boot_work(struct beiscsi_hba
*phba
, unsigned int s_handle
)
4914 if (phba
->boot_struct
.boot_kset
)
4917 /* skip if boot work is already in progress */
4918 if (test_and_set_bit(BEISCSI_HBA_BOOT_WORK
, &phba
->state
))
4921 phba
->boot_struct
.retry
= 3;
4922 phba
->boot_struct
.tag
= 0;
4923 phba
->boot_struct
.s_handle
= s_handle
;
4924 phba
->boot_struct
.action
= BEISCSI_BOOT_GET_SHANDLE
;
4925 schedule_work(&phba
->boot_work
);
4929 * Boot flag info for iscsi-utilities
4930 * Bit 0 Block valid flag
4931 * Bit 1 Firmware booting selected
4933 #define BEISCSI_SYSFS_ISCSI_BOOT_FLAGS 3
4935 static ssize_t
beiscsi_show_boot_tgt_info(void *data
, int type
, char *buf
)
4937 struct beiscsi_hba
*phba
= data
;
4938 struct mgmt_session_info
*boot_sess
= &phba
->boot_struct
.boot_sess
;
4939 struct mgmt_conn_info
*boot_conn
= &boot_sess
->conn_list
[0];
4944 case ISCSI_BOOT_TGT_NAME
:
4945 rc
= sprintf(buf
, "%.*s\n",
4946 (int)strlen(boot_sess
->target_name
),
4947 (char *)&boot_sess
->target_name
);
4949 case ISCSI_BOOT_TGT_IP_ADDR
:
4950 if (boot_conn
->dest_ipaddr
.ip_type
== BEISCSI_IP_TYPE_V4
)
4951 rc
= sprintf(buf
, "%pI4\n",
4952 (char *)&boot_conn
->dest_ipaddr
.addr
);
4954 rc
= sprintf(str
, "%pI6\n",
4955 (char *)&boot_conn
->dest_ipaddr
.addr
);
4957 case ISCSI_BOOT_TGT_PORT
:
4958 rc
= sprintf(str
, "%d\n", boot_conn
->dest_port
);
4961 case ISCSI_BOOT_TGT_CHAP_NAME
:
4962 rc
= sprintf(str
, "%.*s\n",
4963 boot_conn
->negotiated_login_options
.auth_data
.chap
.
4964 target_chap_name_length
,
4965 (char *)&boot_conn
->negotiated_login_options
.
4966 auth_data
.chap
.target_chap_name
);
4968 case ISCSI_BOOT_TGT_CHAP_SECRET
:
4969 rc
= sprintf(str
, "%.*s\n",
4970 boot_conn
->negotiated_login_options
.auth_data
.chap
.
4971 target_secret_length
,
4972 (char *)&boot_conn
->negotiated_login_options
.
4973 auth_data
.chap
.target_secret
);
4975 case ISCSI_BOOT_TGT_REV_CHAP_NAME
:
4976 rc
= sprintf(str
, "%.*s\n",
4977 boot_conn
->negotiated_login_options
.auth_data
.chap
.
4978 intr_chap_name_length
,
4979 (char *)&boot_conn
->negotiated_login_options
.
4980 auth_data
.chap
.intr_chap_name
);
4982 case ISCSI_BOOT_TGT_REV_CHAP_SECRET
:
4983 rc
= sprintf(str
, "%.*s\n",
4984 boot_conn
->negotiated_login_options
.auth_data
.chap
.
4986 (char *)&boot_conn
->negotiated_login_options
.
4987 auth_data
.chap
.intr_secret
);
4989 case ISCSI_BOOT_TGT_FLAGS
:
4990 rc
= sprintf(str
, "%d\n", BEISCSI_SYSFS_ISCSI_BOOT_FLAGS
);
4992 case ISCSI_BOOT_TGT_NIC_ASSOC
:
4993 rc
= sprintf(str
, "0\n");
4999 static ssize_t
beiscsi_show_boot_ini_info(void *data
, int type
, char *buf
)
5001 struct beiscsi_hba
*phba
= data
;
5006 case ISCSI_BOOT_INI_INITIATOR_NAME
:
5007 rc
= sprintf(str
, "%s\n",
5008 phba
->boot_struct
.boot_sess
.initiator_iscsiname
);
5014 static ssize_t
beiscsi_show_boot_eth_info(void *data
, int type
, char *buf
)
5016 struct beiscsi_hba
*phba
= data
;
5021 case ISCSI_BOOT_ETH_FLAGS
:
5022 rc
= sprintf(str
, "%d\n", BEISCSI_SYSFS_ISCSI_BOOT_FLAGS
);
5024 case ISCSI_BOOT_ETH_INDEX
:
5025 rc
= sprintf(str
, "0\n");
5027 case ISCSI_BOOT_ETH_MAC
:
5028 rc
= beiscsi_get_macaddr(str
, phba
);
5034 static umode_t
beiscsi_tgt_get_attr_visibility(void *data
, int type
)
5039 case ISCSI_BOOT_TGT_NAME
:
5040 case ISCSI_BOOT_TGT_IP_ADDR
:
5041 case ISCSI_BOOT_TGT_PORT
:
5042 case ISCSI_BOOT_TGT_CHAP_NAME
:
5043 case ISCSI_BOOT_TGT_CHAP_SECRET
:
5044 case ISCSI_BOOT_TGT_REV_CHAP_NAME
:
5045 case ISCSI_BOOT_TGT_REV_CHAP_SECRET
:
5046 case ISCSI_BOOT_TGT_NIC_ASSOC
:
5047 case ISCSI_BOOT_TGT_FLAGS
:
5054 static umode_t
beiscsi_ini_get_attr_visibility(void *data
, int type
)
5059 case ISCSI_BOOT_INI_INITIATOR_NAME
:
5066 static umode_t
beiscsi_eth_get_attr_visibility(void *data
, int type
)
5071 case ISCSI_BOOT_ETH_FLAGS
:
5072 case ISCSI_BOOT_ETH_MAC
:
5073 case ISCSI_BOOT_ETH_INDEX
:
5080 static void beiscsi_boot_kobj_release(void *data
)
5082 struct beiscsi_hba
*phba
= data
;
5084 scsi_host_put(phba
->shost
);
5087 static int beiscsi_boot_create_kset(struct beiscsi_hba
*phba
)
5089 struct boot_struct
*bs
= &phba
->boot_struct
;
5090 struct iscsi_boot_kobj
*boot_kobj
;
5092 if (bs
->boot_kset
) {
5093 __beiscsi_log(phba
, KERN_ERR
,
5094 "BM_%d: boot_kset already created\n");
5098 bs
->boot_kset
= iscsi_boot_create_host_kset(phba
->shost
->host_no
);
5099 if (!bs
->boot_kset
) {
5100 __beiscsi_log(phba
, KERN_ERR
,
5101 "BM_%d: boot_kset alloc failed\n");
5105 /* get shost ref because the show function will refer phba */
5106 if (!scsi_host_get(phba
->shost
))
5109 boot_kobj
= iscsi_boot_create_target(bs
->boot_kset
, 0, phba
,
5110 beiscsi_show_boot_tgt_info
,
5111 beiscsi_tgt_get_attr_visibility
,
5112 beiscsi_boot_kobj_release
);
5116 if (!scsi_host_get(phba
->shost
))
5119 boot_kobj
= iscsi_boot_create_initiator(bs
->boot_kset
, 0, phba
,
5120 beiscsi_show_boot_ini_info
,
5121 beiscsi_ini_get_attr_visibility
,
5122 beiscsi_boot_kobj_release
);
5126 if (!scsi_host_get(phba
->shost
))
5129 boot_kobj
= iscsi_boot_create_ethernet(bs
->boot_kset
, 0, phba
,
5130 beiscsi_show_boot_eth_info
,
5131 beiscsi_eth_get_attr_visibility
,
5132 beiscsi_boot_kobj_release
);
5139 scsi_host_put(phba
->shost
);
5141 iscsi_boot_destroy_kset(bs
->boot_kset
);
5142 bs
->boot_kset
= NULL
;
5146 static void beiscsi_boot_work(struct work_struct
*work
)
5148 struct beiscsi_hba
*phba
=
5149 container_of(work
, struct beiscsi_hba
, boot_work
);
5150 struct boot_struct
*bs
= &phba
->boot_struct
;
5151 unsigned int tag
= 0;
5153 if (!beiscsi_hba_is_online(phba
))
5156 beiscsi_log(phba
, KERN_INFO
,
5157 BEISCSI_LOG_CONFIG
| BEISCSI_LOG_MBOX
,
5158 "BM_%d : %s action %d\n",
5159 __func__
, phba
->boot_struct
.action
);
5161 switch (phba
->boot_struct
.action
) {
5162 case BEISCSI_BOOT_REOPEN_SESS
:
5163 tag
= beiscsi_boot_reopen_sess(phba
);
5165 case BEISCSI_BOOT_GET_SHANDLE
:
5166 tag
= __beiscsi_boot_get_shandle(phba
, 1);
5168 case BEISCSI_BOOT_GET_SINFO
:
5169 tag
= beiscsi_boot_get_sinfo(phba
);
5171 case BEISCSI_BOOT_LOGOUT_SESS
:
5172 tag
= beiscsi_boot_logout_sess(phba
);
5174 case BEISCSI_BOOT_CREATE_KSET
:
5175 beiscsi_boot_create_kset(phba
);
5177 * updated boot_kset is made visible to all before
5178 * ending the boot work.
5181 clear_bit(BEISCSI_HBA_BOOT_WORK
, &phba
->state
);
5186 schedule_work(&phba
->boot_work
);
5188 clear_bit(BEISCSI_HBA_BOOT_WORK
, &phba
->state
);
5192 static void beiscsi_eqd_update_work(struct work_struct
*work
)
5194 struct hwi_context_memory
*phwi_context
;
5195 struct be_set_eqd set_eqd
[MAX_CPUS
];
5196 struct hwi_controller
*phwi_ctrlr
;
5197 struct be_eq_obj
*pbe_eq
;
5198 struct beiscsi_hba
*phba
;
5199 unsigned int pps
, delta
;
5200 struct be_aic_obj
*aic
;
5201 int eqd
, i
, num
= 0;
5204 phba
= container_of(work
, struct beiscsi_hba
, eqd_update
.work
);
5205 if (!beiscsi_hba_is_online(phba
))
5208 phwi_ctrlr
= phba
->phwi_ctrlr
;
5209 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
5211 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
5212 aic
= &phba
->aic_obj
[i
];
5213 pbe_eq
= &phwi_context
->be_eq
[i
];
5215 if (!aic
->jiffies
|| time_before(now
, aic
->jiffies
) ||
5216 pbe_eq
->cq_count
< aic
->eq_prev
) {
5218 aic
->eq_prev
= pbe_eq
->cq_count
;
5221 delta
= jiffies_to_msecs(now
- aic
->jiffies
);
5222 pps
= (((u32
)(pbe_eq
->cq_count
- aic
->eq_prev
) * 1000) / delta
);
5223 eqd
= (pps
/ 1500) << 2;
5227 eqd
= min_t(u32
, eqd
, BEISCSI_EQ_DELAY_MAX
);
5228 eqd
= max_t(u32
, eqd
, BEISCSI_EQ_DELAY_MIN
);
5231 aic
->eq_prev
= pbe_eq
->cq_count
;
5233 if (eqd
!= aic
->prev_eqd
) {
5234 set_eqd
[num
].delay_multiplier
= (eqd
* 65)/100;
5235 set_eqd
[num
].eq_id
= pbe_eq
->q
.id
;
5236 aic
->prev_eqd
= eqd
;
5241 /* completion of this is ignored */
5242 beiscsi_modify_eq_delay(phba
, set_eqd
, num
);
5244 schedule_delayed_work(&phba
->eqd_update
,
5245 msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL
));
5248 static void beiscsi_hw_tpe_check(struct timer_list
*t
)
5250 struct beiscsi_hba
*phba
= from_timer(phba
, t
, hw_check
);
5253 /* if not TPE, do nothing */
5254 if (!beiscsi_detect_tpe(phba
))
5257 /* wait default 4000ms before recovering */
5259 if (phba
->ue2rp
> BEISCSI_UE_DETECT_INTERVAL
)
5260 wait
= phba
->ue2rp
- BEISCSI_UE_DETECT_INTERVAL
;
5261 queue_delayed_work(phba
->wq
, &phba
->recover_port
,
5262 msecs_to_jiffies(wait
));
5265 static void beiscsi_hw_health_check(struct timer_list
*t
)
5267 struct beiscsi_hba
*phba
= from_timer(phba
, t
, hw_check
);
5269 beiscsi_detect_ue(phba
);
5270 if (beiscsi_detect_ue(phba
)) {
5271 __beiscsi_log(phba
, KERN_ERR
,
5272 "BM_%d : port in error: %lx\n", phba
->state
);
5273 /* sessions are no longer valid, so first fail the sessions */
5274 queue_work(phba
->wq
, &phba
->sess_work
);
5276 /* detect UER supported */
5277 if (!test_bit(BEISCSI_HBA_UER_SUPP
, &phba
->state
))
5279 /* modify this timer to check TPE */
5280 phba
->hw_check
.function
= beiscsi_hw_tpe_check
;
5283 mod_timer(&phba
->hw_check
,
5284 jiffies
+ msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL
));
5288 * beiscsi_enable_port()- Enables the disabled port.
5289 * Only port resources freed in disable function are reallocated.
5290 * This is called in HBA error handling path.
5292 * @phba: Instance of driver private structure
5295 static int beiscsi_enable_port(struct beiscsi_hba
*phba
)
5297 struct hwi_context_memory
*phwi_context
;
5298 struct hwi_controller
*phwi_ctrlr
;
5299 struct be_eq_obj
*pbe_eq
;
5302 if (test_bit(BEISCSI_HBA_ONLINE
, &phba
->state
)) {
5303 __beiscsi_log(phba
, KERN_ERR
,
5304 "BM_%d : %s : port is online %lx\n",
5305 __func__
, phba
->state
);
5309 ret
= beiscsi_init_sliport(phba
);
5313 be2iscsi_enable_msix(phba
);
5315 beiscsi_get_params(phba
);
5316 beiscsi_set_host_data(phba
);
5317 /* Re-enable UER. If different TPE occurs then it is recoverable. */
5318 beiscsi_set_uer_feature(phba
);
5320 phba
->shost
->max_id
= phba
->params
.cxns_per_ctrl
;
5321 phba
->shost
->can_queue
= phba
->params
.ios_per_ctrl
;
5322 ret
= beiscsi_init_port(phba
);
5324 __beiscsi_log(phba
, KERN_ERR
,
5325 "BM_%d : init port failed\n");
5329 for (i
= 0; i
< MAX_MCC_CMD
; i
++) {
5330 init_waitqueue_head(&phba
->ctrl
.mcc_wait
[i
+ 1]);
5331 phba
->ctrl
.mcc_tag
[i
] = i
+ 1;
5332 phba
->ctrl
.mcc_tag_status
[i
+ 1] = 0;
5333 phba
->ctrl
.mcc_tag_available
++;
5336 phwi_ctrlr
= phba
->phwi_ctrlr
;
5337 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
5338 for (i
= 0; i
< phba
->num_cpus
; i
++) {
5339 pbe_eq
= &phwi_context
->be_eq
[i
];
5340 irq_poll_init(&pbe_eq
->iopoll
, be_iopoll_budget
, be_iopoll
);
5343 i
= (phba
->pcidev
->msix_enabled
) ? i
: 0;
5344 /* Work item for MCC handling */
5345 pbe_eq
= &phwi_context
->be_eq
[i
];
5346 INIT_WORK(&pbe_eq
->mcc_work
, beiscsi_mcc_work
);
5348 ret
= beiscsi_init_irqs(phba
);
5350 __beiscsi_log(phba
, KERN_ERR
,
5351 "BM_%d : setup IRQs failed %d\n", ret
);
5354 hwi_enable_intr(phba
);
5355 /* port operational: clear all error bits */
5356 set_bit(BEISCSI_HBA_ONLINE
, &phba
->state
);
5357 __beiscsi_log(phba
, KERN_INFO
,
5358 "BM_%d : port online: 0x%lx\n", phba
->state
);
5360 /* start hw_check timer and eqd_update work */
5361 schedule_delayed_work(&phba
->eqd_update
,
5362 msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL
));
5365 * Timer function gets modified for TPE detection.
5366 * Always reinit to do health check first.
5368 phba
->hw_check
.function
= beiscsi_hw_health_check
;
5369 mod_timer(&phba
->hw_check
,
5370 jiffies
+ msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL
));
5374 for (i
= 0; i
< phba
->num_cpus
; i
++) {
5375 pbe_eq
= &phwi_context
->be_eq
[i
];
5376 irq_poll_disable(&pbe_eq
->iopoll
);
5378 hwi_cleanup_port(phba
);
5381 pci_free_irq_vectors(phba
->pcidev
);
5386 * beiscsi_disable_port()- Disable port and cleanup driver resources.
5387 * This is called in HBA error handling and driver removal.
5388 * @phba: Instance Priv structure
5389 * @unload: indicate driver is unloading
5391 * Free the OS and HW resources held by the driver
5393 static void beiscsi_disable_port(struct beiscsi_hba
*phba
, int unload
)
5395 struct hwi_context_memory
*phwi_context
;
5396 struct hwi_controller
*phwi_ctrlr
;
5397 struct be_eq_obj
*pbe_eq
;
5400 if (!test_and_clear_bit(BEISCSI_HBA_ONLINE
, &phba
->state
))
5403 phwi_ctrlr
= phba
->phwi_ctrlr
;
5404 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
5405 hwi_disable_intr(phba
);
5406 beiscsi_free_irqs(phba
);
5407 pci_free_irq_vectors(phba
->pcidev
);
5409 for (i
= 0; i
< phba
->num_cpus
; i
++) {
5410 pbe_eq
= &phwi_context
->be_eq
[i
];
5411 irq_poll_disable(&pbe_eq
->iopoll
);
5413 cancel_delayed_work_sync(&phba
->eqd_update
);
5414 cancel_work_sync(&phba
->boot_work
);
5415 /* WQ might be running cancel queued mcc_work if we are not exiting */
5416 if (!unload
&& beiscsi_hba_in_error(phba
)) {
5417 pbe_eq
= &phwi_context
->be_eq
[i
];
5418 cancel_work_sync(&pbe_eq
->mcc_work
);
5420 hwi_cleanup_port(phba
);
5421 beiscsi_cleanup_port(phba
);
5424 static void beiscsi_sess_work(struct work_struct
*work
)
5426 struct beiscsi_hba
*phba
;
5428 phba
= container_of(work
, struct beiscsi_hba
, sess_work
);
5430 * This work gets scheduled only in case of HBA error.
5431 * Old sessions are gone so need to be re-established.
5432 * iscsi_session_failure needs process context hence this work.
5434 iscsi_host_for_each_session(phba
->shost
, beiscsi_session_fail
);
5437 static void beiscsi_recover_port(struct work_struct
*work
)
5439 struct beiscsi_hba
*phba
;
5441 phba
= container_of(work
, struct beiscsi_hba
, recover_port
.work
);
5442 beiscsi_disable_port(phba
, 0);
5443 beiscsi_enable_port(phba
);
5446 static pci_ers_result_t
beiscsi_eeh_err_detected(struct pci_dev
*pdev
,
5447 pci_channel_state_t state
)
5449 struct beiscsi_hba
*phba
= NULL
;
5451 phba
= (struct beiscsi_hba
*)pci_get_drvdata(pdev
);
5452 set_bit(BEISCSI_HBA_PCI_ERR
, &phba
->state
);
5454 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5455 "BM_%d : EEH error detected\n");
5457 /* first stop UE detection when PCI error detected */
5458 del_timer_sync(&phba
->hw_check
);
5459 cancel_delayed_work_sync(&phba
->recover_port
);
5461 /* sessions are no longer valid, so first fail the sessions */
5462 iscsi_host_for_each_session(phba
->shost
, beiscsi_session_fail
);
5463 beiscsi_disable_port(phba
, 0);
5465 if (state
== pci_channel_io_perm_failure
) {
5466 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5467 "BM_%d : EEH : State PERM Failure");
5468 return PCI_ERS_RESULT_DISCONNECT
;
5471 pci_disable_device(pdev
);
5473 /* The error could cause the FW to trigger a flash debug dump.
5474 * Resetting the card while flash dump is in progress
5475 * can cause it not to recover; wait for it to finish.
5476 * Wait only for first function as it is needed only once per
5479 if (pdev
->devfn
== 0)
5482 return PCI_ERS_RESULT_NEED_RESET
;
5485 static pci_ers_result_t
beiscsi_eeh_reset(struct pci_dev
*pdev
)
5487 struct beiscsi_hba
*phba
= NULL
;
5490 phba
= (struct beiscsi_hba
*)pci_get_drvdata(pdev
);
5492 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5493 "BM_%d : EEH Reset\n");
5495 status
= pci_enable_device(pdev
);
5497 return PCI_ERS_RESULT_DISCONNECT
;
5499 pci_set_master(pdev
);
5500 pci_set_power_state(pdev
, PCI_D0
);
5501 pci_restore_state(pdev
);
5503 status
= beiscsi_check_fw_rdy(phba
);
5505 beiscsi_log(phba
, KERN_WARNING
, BEISCSI_LOG_INIT
,
5506 "BM_%d : EEH Reset Completed\n");
5508 beiscsi_log(phba
, KERN_WARNING
, BEISCSI_LOG_INIT
,
5509 "BM_%d : EEH Reset Completion Failure\n");
5510 return PCI_ERS_RESULT_DISCONNECT
;
5513 return PCI_ERS_RESULT_RECOVERED
;
5516 static void beiscsi_eeh_resume(struct pci_dev
*pdev
)
5518 struct beiscsi_hba
*phba
;
5521 phba
= (struct beiscsi_hba
*)pci_get_drvdata(pdev
);
5522 pci_save_state(pdev
);
5524 ret
= beiscsi_enable_port(phba
);
5526 __beiscsi_log(phba
, KERN_ERR
,
5527 "BM_%d : AER EEH resume failed\n");
5530 static int beiscsi_dev_probe(struct pci_dev
*pcidev
,
5531 const struct pci_device_id
*id
)
5533 struct hwi_context_memory
*phwi_context
;
5534 struct hwi_controller
*phwi_ctrlr
;
5535 struct beiscsi_hba
*phba
= NULL
;
5536 struct be_eq_obj
*pbe_eq
;
5537 unsigned int s_handle
;
5541 ret
= beiscsi_enable_pci(pcidev
);
5543 dev_err(&pcidev
->dev
,
5544 "beiscsi_dev_probe - Failed to enable pci device\n");
5548 phba
= beiscsi_hba_alloc(pcidev
);
5550 dev_err(&pcidev
->dev
,
5551 "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
5556 /* Enable EEH reporting */
5557 ret
= pci_enable_pcie_error_reporting(pcidev
);
5559 beiscsi_log(phba
, KERN_WARNING
, BEISCSI_LOG_INIT
,
5560 "BM_%d : PCIe Error Reporting "
5561 "Enabling Failed\n");
5563 pci_save_state(pcidev
);
5565 /* Initialize Driver configuration Paramters */
5566 beiscsi_hba_attrs_init(phba
);
5568 phba
->mac_addr_set
= false;
5570 switch (pcidev
->device
) {
5574 phba
->generation
= BE_GEN2
;
5575 phba
->iotask_fn
= beiscsi_iotask
;
5576 dev_warn(&pcidev
->dev
,
5577 "Obsolete/Unsupported BE2 Adapter Family\n");
5581 phba
->generation
= BE_GEN3
;
5582 phba
->iotask_fn
= beiscsi_iotask
;
5585 phba
->generation
= BE_GEN4
;
5586 phba
->iotask_fn
= beiscsi_iotask_v2
;
5589 phba
->generation
= 0;
5592 ret
= be_ctrl_init(phba
, pcidev
);
5594 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5595 "BM_%d : be_ctrl_init failed\n");
5599 ret
= beiscsi_init_sliport(phba
);
5603 spin_lock_init(&phba
->io_sgl_lock
);
5604 spin_lock_init(&phba
->mgmt_sgl_lock
);
5605 spin_lock_init(&phba
->async_pdu_lock
);
5606 ret
= beiscsi_get_fw_config(&phba
->ctrl
, phba
);
5608 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5609 "BM_%d : Error getting fw config\n");
5612 beiscsi_get_port_name(&phba
->ctrl
, phba
);
5613 beiscsi_get_params(phba
);
5614 beiscsi_set_host_data(phba
);
5615 beiscsi_set_uer_feature(phba
);
5617 be2iscsi_enable_msix(phba
);
5619 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
5620 "BM_%d : num_cpus = %d\n",
5623 phba
->shost
->max_id
= phba
->params
.cxns_per_ctrl
;
5624 phba
->shost
->can_queue
= phba
->params
.ios_per_ctrl
;
5625 ret
= beiscsi_get_memory(phba
);
5627 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5628 "BM_%d : alloc host mem failed\n");
5632 ret
= beiscsi_init_port(phba
);
5634 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5635 "BM_%d : init port failed\n");
5636 beiscsi_free_mem(phba
);
5640 for (i
= 0; i
< MAX_MCC_CMD
; i
++) {
5641 init_waitqueue_head(&phba
->ctrl
.mcc_wait
[i
+ 1]);
5642 phba
->ctrl
.mcc_tag
[i
] = i
+ 1;
5643 phba
->ctrl
.mcc_tag_status
[i
+ 1] = 0;
5644 phba
->ctrl
.mcc_tag_available
++;
5645 memset(&phba
->ctrl
.ptag_state
[i
].tag_mem_state
, 0,
5646 sizeof(struct be_dma_mem
));
5649 phba
->ctrl
.mcc_alloc_index
= phba
->ctrl
.mcc_free_index
= 0;
5651 snprintf(wq_name
, sizeof(wq_name
), "beiscsi_%02x_wq",
5652 phba
->shost
->host_no
);
5653 phba
->wq
= alloc_workqueue("%s", WQ_MEM_RECLAIM
, 1, wq_name
);
5655 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5656 "BM_%d : beiscsi_dev_probe-"
5657 "Failed to allocate work queue\n");
5662 INIT_DELAYED_WORK(&phba
->eqd_update
, beiscsi_eqd_update_work
);
5664 phwi_ctrlr
= phba
->phwi_ctrlr
;
5665 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
5667 for (i
= 0; i
< phba
->num_cpus
; i
++) {
5668 pbe_eq
= &phwi_context
->be_eq
[i
];
5669 irq_poll_init(&pbe_eq
->iopoll
, be_iopoll_budget
, be_iopoll
);
5672 i
= (phba
->pcidev
->msix_enabled
) ? i
: 0;
5673 /* Work item for MCC handling */
5674 pbe_eq
= &phwi_context
->be_eq
[i
];
5675 INIT_WORK(&pbe_eq
->mcc_work
, beiscsi_mcc_work
);
5677 ret
= beiscsi_init_irqs(phba
);
5679 beiscsi_log(phba
, KERN_ERR
, BEISCSI_LOG_INIT
,
5680 "BM_%d : beiscsi_dev_probe-"
5681 "Failed to beiscsi_init_irqs\n");
5682 goto disable_iopoll
;
5684 hwi_enable_intr(phba
);
5686 ret
= iscsi_host_add(phba
->shost
, &phba
->pcidev
->dev
);
5690 /* set online bit after port is operational */
5691 set_bit(BEISCSI_HBA_ONLINE
, &phba
->state
);
5692 __beiscsi_log(phba
, KERN_INFO
,
5693 "BM_%d : port online: 0x%lx\n", phba
->state
);
5695 INIT_WORK(&phba
->boot_work
, beiscsi_boot_work
);
5696 ret
= beiscsi_boot_get_shandle(phba
, &s_handle
);
5698 beiscsi_start_boot_work(phba
, s_handle
);
5700 * Set this bit after starting the work to let
5701 * probe handle it first.
5702 * ASYNC event can too schedule this work.
5704 set_bit(BEISCSI_HBA_BOOT_FOUND
, &phba
->state
);
5707 beiscsi_iface_create_default(phba
);
5708 schedule_delayed_work(&phba
->eqd_update
,
5709 msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL
));
5711 INIT_WORK(&phba
->sess_work
, beiscsi_sess_work
);
5712 INIT_DELAYED_WORK(&phba
->recover_port
, beiscsi_recover_port
);
5714 * Start UE detection here. UE before this will cause stall in probe
5715 * and eventually fail the probe.
5717 timer_setup(&phba
->hw_check
, beiscsi_hw_health_check
, 0);
5718 mod_timer(&phba
->hw_check
,
5719 jiffies
+ msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL
));
5720 beiscsi_log(phba
, KERN_INFO
, BEISCSI_LOG_INIT
,
5721 "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
5725 hwi_disable_intr(phba
);
5726 beiscsi_free_irqs(phba
);
5728 for (i
= 0; i
< phba
->num_cpus
; i
++) {
5729 pbe_eq
= &phwi_context
->be_eq
[i
];
5730 irq_poll_disable(&pbe_eq
->iopoll
);
5732 destroy_workqueue(phba
->wq
);
5734 hwi_cleanup_port(phba
);
5735 beiscsi_cleanup_port(phba
);
5736 beiscsi_free_mem(phba
);
5738 dma_free_coherent(&phba
->pcidev
->dev
,
5739 phba
->ctrl
.mbox_mem_alloced
.size
,
5740 phba
->ctrl
.mbox_mem_alloced
.va
,
5741 phba
->ctrl
.mbox_mem_alloced
.dma
);
5742 beiscsi_unmap_pci_function(phba
);
5744 pci_disable_msix(phba
->pcidev
);
5745 pci_dev_put(phba
->pcidev
);
5746 iscsi_host_free(phba
->shost
);
5747 pci_set_drvdata(pcidev
, NULL
);
5749 pci_release_regions(pcidev
);
5750 pci_disable_device(pcidev
);
5754 static void beiscsi_remove(struct pci_dev
*pcidev
)
5756 struct beiscsi_hba
*phba
= NULL
;
5758 phba
= pci_get_drvdata(pcidev
);
5760 dev_err(&pcidev
->dev
, "beiscsi_remove called with no phba\n");
5764 /* first stop UE detection before unloading */
5765 del_timer_sync(&phba
->hw_check
);
5766 cancel_delayed_work_sync(&phba
->recover_port
);
5767 cancel_work_sync(&phba
->sess_work
);
5769 beiscsi_iface_destroy_default(phba
);
5770 iscsi_host_remove(phba
->shost
);
5771 beiscsi_disable_port(phba
, 1);
5773 /* after cancelling boot_work */
5774 iscsi_boot_destroy_kset(phba
->boot_struct
.boot_kset
);
5776 /* free all resources */
5777 destroy_workqueue(phba
->wq
);
5778 beiscsi_free_mem(phba
);
5781 beiscsi_unmap_pci_function(phba
);
5782 dma_free_coherent(&phba
->pcidev
->dev
,
5783 phba
->ctrl
.mbox_mem_alloced
.size
,
5784 phba
->ctrl
.mbox_mem_alloced
.va
,
5785 phba
->ctrl
.mbox_mem_alloced
.dma
);
5787 pci_dev_put(phba
->pcidev
);
5788 iscsi_host_free(phba
->shost
);
5789 pci_disable_pcie_error_reporting(pcidev
);
5790 pci_set_drvdata(pcidev
, NULL
);
5791 pci_release_regions(pcidev
);
5792 pci_disable_device(pcidev
);
5796 static struct pci_error_handlers beiscsi_eeh_handlers
= {
5797 .error_detected
= beiscsi_eeh_err_detected
,
5798 .slot_reset
= beiscsi_eeh_reset
,
5799 .resume
= beiscsi_eeh_resume
,
5802 struct iscsi_transport beiscsi_iscsi_transport
= {
5803 .owner
= THIS_MODULE
,
5805 .caps
= CAP_RECOVERY_L0
| CAP_HDRDGST
| CAP_TEXT_NEGO
|
5806 CAP_MULTI_R2T
| CAP_DATADGST
| CAP_DATA_PATH_OFFLOAD
,
5807 .create_session
= beiscsi_session_create
,
5808 .destroy_session
= beiscsi_session_destroy
,
5809 .create_conn
= beiscsi_conn_create
,
5810 .bind_conn
= beiscsi_conn_bind
,
5811 .destroy_conn
= iscsi_conn_teardown
,
5812 .attr_is_visible
= beiscsi_attr_is_visible
,
5813 .set_iface_param
= beiscsi_iface_set_param
,
5814 .get_iface_param
= beiscsi_iface_get_param
,
5815 .set_param
= beiscsi_set_param
,
5816 .get_conn_param
= iscsi_conn_get_param
,
5817 .get_session_param
= iscsi_session_get_param
,
5818 .get_host_param
= beiscsi_get_host_param
,
5819 .start_conn
= beiscsi_conn_start
,
5820 .stop_conn
= iscsi_conn_stop
,
5821 .send_pdu
= iscsi_conn_send_pdu
,
5822 .xmit_task
= beiscsi_task_xmit
,
5823 .cleanup_task
= beiscsi_cleanup_task
,
5824 .alloc_pdu
= beiscsi_alloc_pdu
,
5825 .parse_pdu_itt
= beiscsi_parse_pdu
,
5826 .get_stats
= beiscsi_conn_get_stats
,
5827 .get_ep_param
= beiscsi_ep_get_param
,
5828 .ep_connect
= beiscsi_ep_connect
,
5829 .ep_poll
= beiscsi_ep_poll
,
5830 .ep_disconnect
= beiscsi_ep_disconnect
,
5831 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
5832 .bsg_request
= beiscsi_bsg_request
,
5835 static struct pci_driver beiscsi_pci_driver
= {
5837 .probe
= beiscsi_dev_probe
,
5838 .remove
= beiscsi_remove
,
5839 .id_table
= beiscsi_pci_id_table
,
5840 .err_handler
= &beiscsi_eeh_handlers
5843 static int __init
beiscsi_module_init(void)
5847 beiscsi_scsi_transport
=
5848 iscsi_register_transport(&beiscsi_iscsi_transport
);
5849 if (!beiscsi_scsi_transport
) {
5851 "beiscsi_module_init - Unable to register beiscsi transport.\n");
5854 printk(KERN_INFO
"In beiscsi_module_init, tt=%p\n",
5855 &beiscsi_iscsi_transport
);
5857 ret
= pci_register_driver(&beiscsi_pci_driver
);
5860 "beiscsi_module_init - Unable to register beiscsi pci driver.\n");
5861 goto unregister_iscsi_transport
;
5865 unregister_iscsi_transport
:
5866 iscsi_unregister_transport(&beiscsi_iscsi_transport
);
5870 static void __exit
beiscsi_module_exit(void)
5872 pci_unregister_driver(&beiscsi_pci_driver
);
5873 iscsi_unregister_transport(&beiscsi_iscsi_transport
);
5876 module_init(beiscsi_module_init
);
5877 module_exit(beiscsi_module_exit
);