]>
Commit | Line | Data |
---|---|---|
fc831825 | 1 | /* QLogic qed NIC Driver |
e8f1cb50 | 2 | * Copyright (c) 2015-2017 QLogic Corporation |
fc831825 | 3 | * |
e8f1cb50 MY |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and /or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
fc831825 YM |
31 | */ |
32 | ||
33 | #include <linux/types.h> | |
34 | #include <asm/byteorder.h> | |
35 | #include <asm/param.h> | |
36 | #include <linux/delay.h> | |
37 | #include <linux/dma-mapping.h> | |
38 | #include <linux/etherdevice.h> | |
39 | #include <linux/interrupt.h> | |
40 | #include <linux/kernel.h> | |
41 | #include <linux/log2.h> | |
42 | #include <linux/module.h> | |
43 | #include <linux/pci.h> | |
44 | #include <linux/slab.h> | |
45 | #include <linux/stddef.h> | |
46 | #include <linux/string.h> | |
47 | #include <linux/version.h> | |
48 | #include <linux/workqueue.h> | |
49 | #include <linux/errno.h> | |
50 | #include <linux/list.h> | |
51 | #include <linux/spinlock.h> | |
52 | #include <linux/qed/qed_iscsi_if.h> | |
53 | #include "qed.h" | |
54 | #include "qed_cxt.h" | |
55 | #include "qed_dev_api.h" | |
56 | #include "qed_hsi.h" | |
57 | #include "qed_hw.h" | |
58 | #include "qed_int.h" | |
59 | #include "qed_iscsi.h" | |
60 | #include "qed_ll2.h" | |
61 | #include "qed_mcp.h" | |
62 | #include "qed_sp.h" | |
63 | #include "qed_sriov.h" | |
64 | #include "qed_reg_addr.h" | |
65 | ||
66 | struct qed_iscsi_conn { | |
67 | struct list_head list_entry; | |
68 | bool free_on_delete; | |
69 | ||
70 | u16 conn_id; | |
71 | u32 icid; | |
72 | u32 fw_cid; | |
73 | ||
74 | u8 layer_code; | |
75 | u8 offl_flags; | |
76 | u8 connect_mode; | |
77 | u32 initial_ack; | |
78 | dma_addr_t sq_pbl_addr; | |
79 | struct qed_chain r2tq; | |
80 | struct qed_chain xhq; | |
81 | struct qed_chain uhq; | |
82 | ||
83 | struct tcp_upload_params *tcp_upload_params_virt_addr; | |
84 | dma_addr_t tcp_upload_params_phys_addr; | |
85 | struct scsi_terminate_extra_params *queue_cnts_virt_addr; | |
86 | dma_addr_t queue_cnts_phys_addr; | |
87 | dma_addr_t syn_phy_addr; | |
88 | ||
89 | u16 syn_ip_payload_length; | |
90 | u8 local_mac[6]; | |
91 | u8 remote_mac[6]; | |
92 | u16 vlan_id; | |
93 | u8 tcp_flags; | |
94 | u8 ip_version; | |
95 | u32 remote_ip[4]; | |
96 | u32 local_ip[4]; | |
97 | u8 ka_max_probe_cnt; | |
98 | u8 dup_ack_theshold; | |
99 | u32 rcv_next; | |
100 | u32 snd_una; | |
101 | u32 snd_next; | |
102 | u32 snd_max; | |
103 | u32 snd_wnd; | |
104 | u32 rcv_wnd; | |
105 | u32 snd_wl1; | |
106 | u32 cwnd; | |
107 | u32 ss_thresh; | |
108 | u16 srtt; | |
109 | u16 rtt_var; | |
110 | u32 ts_time; | |
111 | u32 ts_recent; | |
112 | u32 ts_recent_age; | |
113 | u32 total_rt; | |
114 | u32 ka_timeout_delta; | |
115 | u32 rt_timeout_delta; | |
116 | u8 dup_ack_cnt; | |
117 | u8 snd_wnd_probe_cnt; | |
118 | u8 ka_probe_cnt; | |
119 | u8 rt_cnt; | |
120 | u32 flow_label; | |
121 | u32 ka_timeout; | |
122 | u32 ka_interval; | |
123 | u32 max_rt_time; | |
124 | u32 initial_rcv_wnd; | |
125 | u8 ttl; | |
126 | u8 tos_or_tc; | |
127 | u16 remote_port; | |
128 | u16 local_port; | |
129 | u16 mss; | |
130 | u8 snd_wnd_scale; | |
131 | u8 rcv_wnd_scale; | |
132 | u32 ts_ticks_per_second; | |
133 | u16 da_timeout_value; | |
134 | u8 ack_frequency; | |
135 | ||
136 | u8 update_flag; | |
137 | u8 default_cq; | |
138 | u32 max_seq_size; | |
139 | u32 max_recv_pdu_length; | |
140 | u32 max_send_pdu_length; | |
141 | u32 first_seq_length; | |
142 | u32 exp_stat_sn; | |
143 | u32 stat_sn; | |
144 | u16 physical_q0; | |
145 | u16 physical_q1; | |
146 | u8 abortive_dsconnect; | |
147 | }; | |
148 | ||
149 | static int | |
150 | qed_sp_iscsi_func_start(struct qed_hwfn *p_hwfn, | |
151 | enum spq_mode comp_mode, | |
152 | struct qed_spq_comp_cb *p_comp_addr, | |
153 | void *event_context, iscsi_event_cb_t async_event_cb) | |
154 | { | |
155 | struct iscsi_init_ramrod_params *p_ramrod = NULL; | |
156 | struct scsi_init_func_queues *p_queue = NULL; | |
157 | struct qed_iscsi_pf_params *p_params = NULL; | |
158 | struct iscsi_spe_func_init *p_init = NULL; | |
159 | struct qed_spq_entry *p_ent = NULL; | |
160 | struct qed_sp_init_data init_data; | |
161 | int rc = 0; | |
162 | u32 dval; | |
163 | u16 val; | |
164 | u8 i; | |
165 | ||
166 | /* Get SPQ entry */ | |
167 | memset(&init_data, 0, sizeof(init_data)); | |
168 | init_data.cid = qed_spq_get_cid(p_hwfn); | |
169 | init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; | |
170 | init_data.comp_mode = comp_mode; | |
171 | init_data.p_comp_data = p_comp_addr; | |
172 | ||
173 | rc = qed_sp_init_request(p_hwfn, &p_ent, | |
174 | ISCSI_RAMROD_CMD_ID_INIT_FUNC, | |
175 | PROTOCOLID_ISCSI, &init_data); | |
176 | if (rc) | |
177 | return rc; | |
178 | ||
179 | p_ramrod = &p_ent->ramrod.iscsi_init; | |
180 | p_init = &p_ramrod->iscsi_init_spe; | |
181 | p_params = &p_hwfn->pf_params.iscsi_pf_params; | |
182 | p_queue = &p_init->q_params; | |
183 | ||
184 | SET_FIELD(p_init->hdr.flags, | |
185 | ISCSI_SLOW_PATH_HDR_LAYER_CODE, ISCSI_SLOW_PATH_LAYER_CODE); | |
186 | p_init->hdr.op_code = ISCSI_RAMROD_CMD_ID_INIT_FUNC; | |
187 | ||
188 | val = p_params->half_way_close_timeout; | |
189 | p_init->half_way_close_timeout = cpu_to_le16(val); | |
190 | p_init->num_sq_pages_in_ring = p_params->num_sq_pages_in_ring; | |
191 | p_init->num_r2tq_pages_in_ring = p_params->num_r2tq_pages_in_ring; | |
192 | p_init->num_uhq_pages_in_ring = p_params->num_uhq_pages_in_ring; | |
6b116b1d MY |
193 | p_init->ooo_enable = p_params->ooo_enable; |
194 | p_init->ll2_rx_queue_id = p_hwfn->hw_info.resc_start[QED_LL2_QUEUE] + | |
195 | p_params->ll2_ooo_queue_id; | |
fc831825 YM |
196 | p_init->func_params.log_page_size = p_params->log_page_size; |
197 | val = p_params->num_tasks; | |
198 | p_init->func_params.num_tasks = cpu_to_le16(val); | |
199 | p_init->debug_mode.flags = p_params->debug_mode; | |
200 | ||
201 | DMA_REGPAIR_LE(p_queue->glbl_q_params_addr, | |
202 | p_params->glbl_q_params_addr); | |
203 | ||
204 | val = p_params->cq_num_entries; | |
205 | p_queue->cq_num_entries = cpu_to_le16(val); | |
206 | val = p_params->cmdq_num_entries; | |
207 | p_queue->cmdq_num_entries = cpu_to_le16(val); | |
208 | p_queue->num_queues = p_params->num_queues; | |
209 | dval = (u8)p_hwfn->hw_info.resc_start[QED_CMDQS_CQS]; | |
210 | p_queue->queue_relative_offset = (u8)dval; | |
211 | p_queue->cq_sb_pi = p_params->gl_rq_pi; | |
212 | p_queue->cmdq_sb_pi = p_params->gl_cmd_pi; | |
213 | ||
214 | for (i = 0; i < p_params->num_queues; i++) { | |
215 | val = p_hwfn->sbs_info[i]->igu_sb_id; | |
216 | p_queue->cq_cmdq_sb_num_arr[i] = cpu_to_le16(val); | |
217 | } | |
218 | ||
d0d40a73 | 219 | p_queue->bdq_resource_id = (u8)RESC_START(p_hwfn, QED_BDQ); |
fc831825 YM |
220 | |
221 | DMA_REGPAIR_LE(p_queue->bdq_pbl_base_address[BDQ_ID_RQ], | |
222 | p_params->bdq_pbl_base_addr[BDQ_ID_RQ]); | |
223 | p_queue->bdq_pbl_num_entries[BDQ_ID_RQ] = | |
224 | p_params->bdq_pbl_num_entries[BDQ_ID_RQ]; | |
225 | val = p_params->bdq_xoff_threshold[BDQ_ID_RQ]; | |
226 | p_queue->bdq_xoff_threshold[BDQ_ID_RQ] = cpu_to_le16(val); | |
227 | val = p_params->bdq_xon_threshold[BDQ_ID_RQ]; | |
228 | p_queue->bdq_xon_threshold[BDQ_ID_RQ] = cpu_to_le16(val); | |
229 | ||
230 | DMA_REGPAIR_LE(p_queue->bdq_pbl_base_address[BDQ_ID_IMM_DATA], | |
231 | p_params->bdq_pbl_base_addr[BDQ_ID_IMM_DATA]); | |
232 | p_queue->bdq_pbl_num_entries[BDQ_ID_IMM_DATA] = | |
233 | p_params->bdq_pbl_num_entries[BDQ_ID_IMM_DATA]; | |
234 | val = p_params->bdq_xoff_threshold[BDQ_ID_IMM_DATA]; | |
235 | p_queue->bdq_xoff_threshold[BDQ_ID_IMM_DATA] = cpu_to_le16(val); | |
236 | val = p_params->bdq_xon_threshold[BDQ_ID_IMM_DATA]; | |
237 | p_queue->bdq_xon_threshold[BDQ_ID_IMM_DATA] = cpu_to_le16(val); | |
238 | val = p_params->rq_buffer_size; | |
239 | p_queue->rq_buffer_size = cpu_to_le16(val); | |
240 | if (p_params->is_target) { | |
241 | SET_FIELD(p_queue->q_validity, | |
242 | SCSI_INIT_FUNC_QUEUES_RQ_VALID, 1); | |
243 | if (p_queue->bdq_pbl_num_entries[BDQ_ID_IMM_DATA]) | |
244 | SET_FIELD(p_queue->q_validity, | |
245 | SCSI_INIT_FUNC_QUEUES_IMM_DATA_VALID, 1); | |
246 | SET_FIELD(p_queue->q_validity, | |
247 | SCSI_INIT_FUNC_QUEUES_CMD_VALID, 1); | |
248 | } else { | |
249 | SET_FIELD(p_queue->q_validity, | |
250 | SCSI_INIT_FUNC_QUEUES_RQ_VALID, 1); | |
251 | } | |
252 | p_ramrod->tcp_init.two_msl_timer = cpu_to_le32(p_params->two_msl_timer); | |
253 | val = p_params->tx_sws_timer; | |
254 | p_ramrod->tcp_init.tx_sws_timer = cpu_to_le16(val); | |
255 | p_ramrod->tcp_init.maxfinrt = p_params->max_fin_rt; | |
256 | ||
257 | p_hwfn->p_iscsi_info->event_context = event_context; | |
258 | p_hwfn->p_iscsi_info->event_cb = async_event_cb; | |
259 | ||
260 | return qed_spq_post(p_hwfn, p_ent, NULL); | |
261 | } | |
262 | ||
263 | static int qed_sp_iscsi_conn_offload(struct qed_hwfn *p_hwfn, | |
264 | struct qed_iscsi_conn *p_conn, | |
265 | enum spq_mode comp_mode, | |
266 | struct qed_spq_comp_cb *p_comp_addr) | |
267 | { | |
268 | struct iscsi_spe_conn_offload *p_ramrod = NULL; | |
269 | struct tcp_offload_params_opt2 *p_tcp2 = NULL; | |
270 | struct tcp_offload_params *p_tcp = NULL; | |
271 | struct qed_spq_entry *p_ent = NULL; | |
272 | struct qed_sp_init_data init_data; | |
fc831825 YM |
273 | dma_addr_t r2tq_pbl_addr; |
274 | dma_addr_t xhq_pbl_addr; | |
275 | dma_addr_t uhq_pbl_addr; | |
b5a9ee7c | 276 | u16 physical_q; |
fc831825 YM |
277 | int rc = 0; |
278 | u32 dval; | |
279 | u16 wval; | |
280 | u8 i; | |
281 | u16 *p; | |
282 | ||
283 | /* Get SPQ entry */ | |
284 | memset(&init_data, 0, sizeof(init_data)); | |
285 | init_data.cid = p_conn->icid; | |
286 | init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; | |
287 | init_data.comp_mode = comp_mode; | |
288 | init_data.p_comp_data = p_comp_addr; | |
289 | ||
290 | rc = qed_sp_init_request(p_hwfn, &p_ent, | |
291 | ISCSI_RAMROD_CMD_ID_OFFLOAD_CONN, | |
292 | PROTOCOLID_ISCSI, &init_data); | |
293 | if (rc) | |
294 | return rc; | |
295 | ||
296 | p_ramrod = &p_ent->ramrod.iscsi_conn_offload; | |
297 | ||
298 | /* Transmission PQ is the first of the PF */ | |
b5a9ee7c AE |
299 | physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); |
300 | p_conn->physical_q0 = cpu_to_le16(physical_q); | |
301 | p_ramrod->iscsi.physical_q0 = cpu_to_le16(physical_q); | |
fc831825 YM |
302 | |
303 | /* iSCSI Pure-ACK PQ */ | |
b5a9ee7c AE |
304 | physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK); |
305 | p_conn->physical_q1 = cpu_to_le16(physical_q); | |
306 | p_ramrod->iscsi.physical_q1 = cpu_to_le16(physical_q); | |
fc831825 YM |
307 | |
308 | p_ramrod->hdr.op_code = ISCSI_RAMROD_CMD_ID_OFFLOAD_CONN; | |
309 | SET_FIELD(p_ramrod->hdr.flags, ISCSI_SLOW_PATH_HDR_LAYER_CODE, | |
310 | p_conn->layer_code); | |
311 | ||
312 | p_ramrod->conn_id = cpu_to_le16(p_conn->conn_id); | |
313 | p_ramrod->fw_cid = cpu_to_le32(p_conn->icid); | |
314 | ||
315 | DMA_REGPAIR_LE(p_ramrod->iscsi.sq_pbl_addr, p_conn->sq_pbl_addr); | |
316 | ||
317 | r2tq_pbl_addr = qed_chain_get_pbl_phys(&p_conn->r2tq); | |
318 | DMA_REGPAIR_LE(p_ramrod->iscsi.r2tq_pbl_addr, r2tq_pbl_addr); | |
319 | ||
320 | xhq_pbl_addr = qed_chain_get_pbl_phys(&p_conn->xhq); | |
321 | DMA_REGPAIR_LE(p_ramrod->iscsi.xhq_pbl_addr, xhq_pbl_addr); | |
322 | ||
323 | uhq_pbl_addr = qed_chain_get_pbl_phys(&p_conn->uhq); | |
324 | DMA_REGPAIR_LE(p_ramrod->iscsi.uhq_pbl_addr, uhq_pbl_addr); | |
325 | ||
326 | p_ramrod->iscsi.initial_ack = cpu_to_le32(p_conn->initial_ack); | |
327 | p_ramrod->iscsi.flags = p_conn->offl_flags; | |
328 | p_ramrod->iscsi.default_cq = p_conn->default_cq; | |
329 | p_ramrod->iscsi.stat_sn = cpu_to_le32(p_conn->stat_sn); | |
330 | ||
331 | if (!GET_FIELD(p_ramrod->iscsi.flags, | |
332 | ISCSI_CONN_OFFLOAD_PARAMS_TCP_ON_CHIP_1B)) { | |
333 | p_tcp = &p_ramrod->tcp; | |
334 | ||
335 | p = (u16 *)p_conn->local_mac; | |
336 | p_tcp->local_mac_addr_hi = swab16(get_unaligned(p)); | |
337 | p_tcp->local_mac_addr_mid = swab16(get_unaligned(p + 1)); | |
338 | p_tcp->local_mac_addr_lo = swab16(get_unaligned(p + 2)); | |
339 | ||
340 | p = (u16 *)p_conn->remote_mac; | |
341 | p_tcp->remote_mac_addr_hi = swab16(get_unaligned(p)); | |
342 | p_tcp->remote_mac_addr_mid = swab16(get_unaligned(p + 1)); | |
343 | p_tcp->remote_mac_addr_lo = swab16(get_unaligned(p + 2)); | |
344 | ||
345 | p_tcp->vlan_id = cpu_to_le16(p_conn->vlan_id); | |
346 | ||
347 | p_tcp->flags = p_conn->tcp_flags; | |
348 | p_tcp->ip_version = p_conn->ip_version; | |
349 | for (i = 0; i < 4; i++) { | |
350 | dval = p_conn->remote_ip[i]; | |
351 | p_tcp->remote_ip[i] = cpu_to_le32(dval); | |
352 | dval = p_conn->local_ip[i]; | |
353 | p_tcp->local_ip[i] = cpu_to_le32(dval); | |
354 | } | |
355 | p_tcp->ka_max_probe_cnt = p_conn->ka_max_probe_cnt; | |
356 | p_tcp->dup_ack_theshold = p_conn->dup_ack_theshold; | |
357 | ||
358 | p_tcp->rcv_next = cpu_to_le32(p_conn->rcv_next); | |
359 | p_tcp->snd_una = cpu_to_le32(p_conn->snd_una); | |
360 | p_tcp->snd_next = cpu_to_le32(p_conn->snd_next); | |
361 | p_tcp->snd_max = cpu_to_le32(p_conn->snd_max); | |
362 | p_tcp->snd_wnd = cpu_to_le32(p_conn->snd_wnd); | |
363 | p_tcp->rcv_wnd = cpu_to_le32(p_conn->rcv_wnd); | |
364 | p_tcp->snd_wl1 = cpu_to_le32(p_conn->snd_wl1); | |
365 | p_tcp->cwnd = cpu_to_le32(p_conn->cwnd); | |
366 | p_tcp->ss_thresh = cpu_to_le32(p_conn->ss_thresh); | |
367 | p_tcp->srtt = cpu_to_le16(p_conn->srtt); | |
368 | p_tcp->rtt_var = cpu_to_le16(p_conn->rtt_var); | |
369 | p_tcp->ts_time = cpu_to_le32(p_conn->ts_time); | |
370 | p_tcp->ts_recent = cpu_to_le32(p_conn->ts_recent); | |
371 | p_tcp->ts_recent_age = cpu_to_le32(p_conn->ts_recent_age); | |
372 | p_tcp->total_rt = cpu_to_le32(p_conn->total_rt); | |
373 | dval = p_conn->ka_timeout_delta; | |
374 | p_tcp->ka_timeout_delta = cpu_to_le32(dval); | |
375 | dval = p_conn->rt_timeout_delta; | |
376 | p_tcp->rt_timeout_delta = cpu_to_le32(dval); | |
377 | p_tcp->dup_ack_cnt = p_conn->dup_ack_cnt; | |
378 | p_tcp->snd_wnd_probe_cnt = p_conn->snd_wnd_probe_cnt; | |
379 | p_tcp->ka_probe_cnt = p_conn->ka_probe_cnt; | |
380 | p_tcp->rt_cnt = p_conn->rt_cnt; | |
381 | p_tcp->flow_label = cpu_to_le32(p_conn->flow_label); | |
382 | p_tcp->ka_timeout = cpu_to_le32(p_conn->ka_timeout); | |
383 | p_tcp->ka_interval = cpu_to_le32(p_conn->ka_interval); | |
384 | p_tcp->max_rt_time = cpu_to_le32(p_conn->max_rt_time); | |
385 | dval = p_conn->initial_rcv_wnd; | |
386 | p_tcp->initial_rcv_wnd = cpu_to_le32(dval); | |
387 | p_tcp->ttl = p_conn->ttl; | |
388 | p_tcp->tos_or_tc = p_conn->tos_or_tc; | |
389 | p_tcp->remote_port = cpu_to_le16(p_conn->remote_port); | |
390 | p_tcp->local_port = cpu_to_le16(p_conn->local_port); | |
391 | p_tcp->mss = cpu_to_le16(p_conn->mss); | |
392 | p_tcp->snd_wnd_scale = p_conn->snd_wnd_scale; | |
393 | p_tcp->rcv_wnd_scale = p_conn->rcv_wnd_scale; | |
394 | dval = p_conn->ts_ticks_per_second; | |
395 | p_tcp->ts_ticks_per_second = cpu_to_le32(dval); | |
396 | wval = p_conn->da_timeout_value; | |
397 | p_tcp->da_timeout_value = cpu_to_le16(wval); | |
398 | p_tcp->ack_frequency = p_conn->ack_frequency; | |
399 | p_tcp->connect_mode = p_conn->connect_mode; | |
400 | } else { | |
401 | p_tcp2 = | |
402 | &((struct iscsi_spe_conn_offload_option2 *)p_ramrod)->tcp; | |
403 | ||
404 | p = (u16 *)p_conn->local_mac; | |
405 | p_tcp2->local_mac_addr_hi = swab16(get_unaligned(p)); | |
406 | p_tcp2->local_mac_addr_mid = swab16(get_unaligned(p + 1)); | |
407 | p_tcp2->local_mac_addr_lo = swab16(get_unaligned(p + 2)); | |
408 | ||
409 | p = (u16 *)p_conn->remote_mac; | |
410 | p_tcp2->remote_mac_addr_hi = swab16(get_unaligned(p)); | |
411 | p_tcp2->remote_mac_addr_mid = swab16(get_unaligned(p + 1)); | |
412 | p_tcp2->remote_mac_addr_lo = swab16(get_unaligned(p + 2)); | |
413 | ||
414 | p_tcp2->vlan_id = cpu_to_le16(p_conn->vlan_id); | |
415 | p_tcp2->flags = p_conn->tcp_flags; | |
416 | ||
417 | p_tcp2->ip_version = p_conn->ip_version; | |
418 | for (i = 0; i < 4; i++) { | |
419 | dval = p_conn->remote_ip[i]; | |
420 | p_tcp2->remote_ip[i] = cpu_to_le32(dval); | |
421 | dval = p_conn->local_ip[i]; | |
422 | p_tcp2->local_ip[i] = cpu_to_le32(dval); | |
423 | } | |
424 | ||
425 | p_tcp2->flow_label = cpu_to_le32(p_conn->flow_label); | |
426 | p_tcp2->ttl = p_conn->ttl; | |
427 | p_tcp2->tos_or_tc = p_conn->tos_or_tc; | |
428 | p_tcp2->remote_port = cpu_to_le16(p_conn->remote_port); | |
429 | p_tcp2->local_port = cpu_to_le16(p_conn->local_port); | |
430 | p_tcp2->mss = cpu_to_le16(p_conn->mss); | |
431 | p_tcp2->rcv_wnd_scale = p_conn->rcv_wnd_scale; | |
432 | p_tcp2->connect_mode = p_conn->connect_mode; | |
433 | wval = p_conn->syn_ip_payload_length; | |
434 | p_tcp2->syn_ip_payload_length = cpu_to_le16(wval); | |
435 | p_tcp2->syn_phy_addr_lo = DMA_LO_LE(p_conn->syn_phy_addr); | |
436 | p_tcp2->syn_phy_addr_hi = DMA_HI_LE(p_conn->syn_phy_addr); | |
437 | } | |
438 | ||
439 | return qed_spq_post(p_hwfn, p_ent, NULL); | |
440 | } | |
441 | ||
442 | static int qed_sp_iscsi_conn_update(struct qed_hwfn *p_hwfn, | |
443 | struct qed_iscsi_conn *p_conn, | |
444 | enum spq_mode comp_mode, | |
445 | struct qed_spq_comp_cb *p_comp_addr) | |
446 | { | |
447 | struct iscsi_conn_update_ramrod_params *p_ramrod = NULL; | |
448 | struct qed_spq_entry *p_ent = NULL; | |
449 | struct qed_sp_init_data init_data; | |
450 | int rc = -EINVAL; | |
451 | u32 dval; | |
452 | ||
453 | /* Get SPQ entry */ | |
454 | memset(&init_data, 0, sizeof(init_data)); | |
455 | init_data.cid = p_conn->icid; | |
456 | init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; | |
457 | init_data.comp_mode = comp_mode; | |
458 | init_data.p_comp_data = p_comp_addr; | |
459 | ||
460 | rc = qed_sp_init_request(p_hwfn, &p_ent, | |
461 | ISCSI_RAMROD_CMD_ID_UPDATE_CONN, | |
462 | PROTOCOLID_ISCSI, &init_data); | |
463 | if (rc) | |
464 | return rc; | |
465 | ||
466 | p_ramrod = &p_ent->ramrod.iscsi_conn_update; | |
467 | p_ramrod->hdr.op_code = ISCSI_RAMROD_CMD_ID_UPDATE_CONN; | |
468 | SET_FIELD(p_ramrod->hdr.flags, | |
469 | ISCSI_SLOW_PATH_HDR_LAYER_CODE, p_conn->layer_code); | |
470 | ||
471 | p_ramrod->conn_id = cpu_to_le16(p_conn->conn_id); | |
472 | p_ramrod->fw_cid = cpu_to_le32(p_conn->icid); | |
473 | p_ramrod->flags = p_conn->update_flag; | |
474 | p_ramrod->max_seq_size = cpu_to_le32(p_conn->max_seq_size); | |
475 | dval = p_conn->max_recv_pdu_length; | |
476 | p_ramrod->max_recv_pdu_length = cpu_to_le32(dval); | |
477 | dval = p_conn->max_send_pdu_length; | |
478 | p_ramrod->max_send_pdu_length = cpu_to_le32(dval); | |
479 | dval = p_conn->first_seq_length; | |
480 | p_ramrod->first_seq_length = cpu_to_le32(dval); | |
481 | p_ramrod->exp_stat_sn = cpu_to_le32(p_conn->exp_stat_sn); | |
482 | ||
483 | return qed_spq_post(p_hwfn, p_ent, NULL); | |
484 | } | |
485 | ||
486 | static int qed_sp_iscsi_conn_terminate(struct qed_hwfn *p_hwfn, | |
487 | struct qed_iscsi_conn *p_conn, | |
488 | enum spq_mode comp_mode, | |
489 | struct qed_spq_comp_cb *p_comp_addr) | |
490 | { | |
491 | struct iscsi_spe_conn_termination *p_ramrod = NULL; | |
492 | struct qed_spq_entry *p_ent = NULL; | |
493 | struct qed_sp_init_data init_data; | |
494 | int rc = -EINVAL; | |
495 | ||
496 | /* Get SPQ entry */ | |
497 | memset(&init_data, 0, sizeof(init_data)); | |
498 | init_data.cid = p_conn->icid; | |
499 | init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; | |
500 | init_data.comp_mode = comp_mode; | |
501 | init_data.p_comp_data = p_comp_addr; | |
502 | ||
503 | rc = qed_sp_init_request(p_hwfn, &p_ent, | |
504 | ISCSI_RAMROD_CMD_ID_TERMINATION_CONN, | |
505 | PROTOCOLID_ISCSI, &init_data); | |
506 | if (rc) | |
507 | return rc; | |
508 | ||
509 | p_ramrod = &p_ent->ramrod.iscsi_conn_terminate; | |
510 | p_ramrod->hdr.op_code = ISCSI_RAMROD_CMD_ID_TERMINATION_CONN; | |
511 | SET_FIELD(p_ramrod->hdr.flags, | |
512 | ISCSI_SLOW_PATH_HDR_LAYER_CODE, p_conn->layer_code); | |
513 | ||
514 | p_ramrod->conn_id = cpu_to_le16(p_conn->conn_id); | |
515 | p_ramrod->fw_cid = cpu_to_le32(p_conn->icid); | |
516 | p_ramrod->abortive = p_conn->abortive_dsconnect; | |
517 | ||
518 | DMA_REGPAIR_LE(p_ramrod->query_params_addr, | |
519 | p_conn->tcp_upload_params_phys_addr); | |
520 | DMA_REGPAIR_LE(p_ramrod->queue_cnts_addr, p_conn->queue_cnts_phys_addr); | |
521 | ||
522 | return qed_spq_post(p_hwfn, p_ent, NULL); | |
523 | } | |
524 | ||
525 | static int qed_sp_iscsi_conn_clear_sq(struct qed_hwfn *p_hwfn, | |
526 | struct qed_iscsi_conn *p_conn, | |
527 | enum spq_mode comp_mode, | |
528 | struct qed_spq_comp_cb *p_comp_addr) | |
529 | { | |
530 | struct iscsi_slow_path_hdr *p_ramrod = NULL; | |
531 | struct qed_spq_entry *p_ent = NULL; | |
532 | struct qed_sp_init_data init_data; | |
533 | int rc = -EINVAL; | |
534 | ||
535 | /* Get SPQ entry */ | |
536 | memset(&init_data, 0, sizeof(init_data)); | |
537 | init_data.cid = p_conn->icid; | |
538 | init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; | |
539 | init_data.comp_mode = comp_mode; | |
540 | init_data.p_comp_data = p_comp_addr; | |
541 | ||
542 | rc = qed_sp_init_request(p_hwfn, &p_ent, | |
543 | ISCSI_RAMROD_CMD_ID_CLEAR_SQ, | |
544 | PROTOCOLID_ISCSI, &init_data); | |
545 | if (rc) | |
546 | return rc; | |
547 | ||
548 | p_ramrod = &p_ent->ramrod.iscsi_empty; | |
549 | p_ramrod->op_code = ISCSI_RAMROD_CMD_ID_CLEAR_SQ; | |
550 | SET_FIELD(p_ramrod->flags, | |
551 | ISCSI_SLOW_PATH_HDR_LAYER_CODE, p_conn->layer_code); | |
552 | ||
553 | return qed_spq_post(p_hwfn, p_ent, NULL); | |
554 | } | |
555 | ||
556 | static int qed_sp_iscsi_func_stop(struct qed_hwfn *p_hwfn, | |
557 | enum spq_mode comp_mode, | |
558 | struct qed_spq_comp_cb *p_comp_addr) | |
559 | { | |
560 | struct iscsi_spe_func_dstry *p_ramrod = NULL; | |
561 | struct qed_spq_entry *p_ent = NULL; | |
562 | struct qed_sp_init_data init_data; | |
563 | int rc = 0; | |
564 | ||
565 | /* Get SPQ entry */ | |
566 | memset(&init_data, 0, sizeof(init_data)); | |
567 | init_data.cid = qed_spq_get_cid(p_hwfn); | |
568 | init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; | |
569 | init_data.comp_mode = comp_mode; | |
570 | init_data.p_comp_data = p_comp_addr; | |
571 | ||
572 | rc = qed_sp_init_request(p_hwfn, &p_ent, | |
573 | ISCSI_RAMROD_CMD_ID_DESTROY_FUNC, | |
574 | PROTOCOLID_ISCSI, &init_data); | |
575 | if (rc) | |
576 | return rc; | |
577 | ||
578 | p_ramrod = &p_ent->ramrod.iscsi_destroy; | |
579 | p_ramrod->hdr.op_code = ISCSI_RAMROD_CMD_ID_DESTROY_FUNC; | |
580 | ||
581 | return qed_spq_post(p_hwfn, p_ent, NULL); | |
582 | } | |
583 | ||
584 | static void __iomem *qed_iscsi_get_db_addr(struct qed_hwfn *p_hwfn, u32 cid) | |
585 | { | |
586 | return (u8 __iomem *)p_hwfn->doorbells + | |
587 | qed_db_addr(cid, DQ_DEMS_LEGACY); | |
588 | } | |
589 | ||
590 | static void __iomem *qed_iscsi_get_primary_bdq_prod(struct qed_hwfn *p_hwfn, | |
591 | u8 bdq_id) | |
592 | { | |
d0d40a73 MY |
593 | if (RESC_NUM(p_hwfn, QED_BDQ)) { |
594 | return (u8 __iomem *)p_hwfn->regview + | |
595 | GTT_BAR0_MAP_REG_MSDM_RAM + | |
596 | MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(RESC_START(p_hwfn, | |
597 | QED_BDQ), | |
598 | bdq_id); | |
599 | } else { | |
600 | DP_NOTICE(p_hwfn, "BDQ is not allocated!\n"); | |
601 | return NULL; | |
602 | } | |
fc831825 YM |
603 | } |
604 | ||
605 | static void __iomem *qed_iscsi_get_secondary_bdq_prod(struct qed_hwfn *p_hwfn, | |
606 | u8 bdq_id) | |
607 | { | |
d0d40a73 MY |
608 | if (RESC_NUM(p_hwfn, QED_BDQ)) { |
609 | return (u8 __iomem *)p_hwfn->regview + | |
610 | GTT_BAR0_MAP_REG_TSDM_RAM + | |
611 | TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(RESC_START(p_hwfn, | |
612 | QED_BDQ), | |
613 | bdq_id); | |
614 | } else { | |
615 | DP_NOTICE(p_hwfn, "BDQ is not allocated!\n"); | |
616 | return NULL; | |
617 | } | |
fc831825 YM |
618 | } |
619 | ||
620 | static int qed_iscsi_setup_connection(struct qed_hwfn *p_hwfn, | |
621 | struct qed_iscsi_conn *p_conn) | |
622 | { | |
623 | if (!p_conn->queue_cnts_virt_addr) | |
624 | goto nomem; | |
625 | memset(p_conn->queue_cnts_virt_addr, 0, | |
626 | sizeof(*p_conn->queue_cnts_virt_addr)); | |
627 | ||
628 | if (!p_conn->tcp_upload_params_virt_addr) | |
629 | goto nomem; | |
630 | memset(p_conn->tcp_upload_params_virt_addr, 0, | |
631 | sizeof(*p_conn->tcp_upload_params_virt_addr)); | |
632 | ||
633 | if (!p_conn->r2tq.p_virt_addr) | |
634 | goto nomem; | |
635 | qed_chain_pbl_zero_mem(&p_conn->r2tq); | |
636 | ||
637 | if (!p_conn->uhq.p_virt_addr) | |
638 | goto nomem; | |
639 | qed_chain_pbl_zero_mem(&p_conn->uhq); | |
640 | ||
641 | if (!p_conn->xhq.p_virt_addr) | |
642 | goto nomem; | |
643 | qed_chain_pbl_zero_mem(&p_conn->xhq); | |
644 | ||
645 | return 0; | |
646 | nomem: | |
647 | return -ENOMEM; | |
648 | } | |
649 | ||
650 | static int qed_iscsi_allocate_connection(struct qed_hwfn *p_hwfn, | |
651 | struct qed_iscsi_conn **p_out_conn) | |
652 | { | |
653 | u16 uhq_num_elements = 0, xhq_num_elements = 0, r2tq_num_elements = 0; | |
654 | struct scsi_terminate_extra_params *p_q_cnts = NULL; | |
655 | struct qed_iscsi_pf_params *p_params = NULL; | |
656 | struct tcp_upload_params *p_tcp = NULL; | |
657 | struct qed_iscsi_conn *p_conn = NULL; | |
658 | int rc = 0; | |
659 | ||
660 | /* Try finding a free connection that can be used */ | |
661 | spin_lock_bh(&p_hwfn->p_iscsi_info->lock); | |
662 | if (!list_empty(&p_hwfn->p_iscsi_info->free_list)) | |
663 | p_conn = list_first_entry(&p_hwfn->p_iscsi_info->free_list, | |
664 | struct qed_iscsi_conn, list_entry); | |
665 | if (p_conn) { | |
666 | list_del(&p_conn->list_entry); | |
667 | spin_unlock_bh(&p_hwfn->p_iscsi_info->lock); | |
668 | *p_out_conn = p_conn; | |
669 | return 0; | |
670 | } | |
671 | spin_unlock_bh(&p_hwfn->p_iscsi_info->lock); | |
672 | ||
673 | /* Need to allocate a new connection */ | |
674 | p_params = &p_hwfn->pf_params.iscsi_pf_params; | |
675 | ||
676 | p_conn = kzalloc(sizeof(*p_conn), GFP_KERNEL); | |
677 | if (!p_conn) | |
678 | return -ENOMEM; | |
679 | ||
680 | p_q_cnts = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, | |
681 | sizeof(*p_q_cnts), | |
682 | &p_conn->queue_cnts_phys_addr, | |
683 | GFP_KERNEL); | |
684 | if (!p_q_cnts) | |
685 | goto nomem_queue_cnts_param; | |
686 | p_conn->queue_cnts_virt_addr = p_q_cnts; | |
687 | ||
688 | p_tcp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, | |
689 | sizeof(*p_tcp), | |
690 | &p_conn->tcp_upload_params_phys_addr, | |
691 | GFP_KERNEL); | |
692 | if (!p_tcp) | |
693 | goto nomem_upload_param; | |
694 | p_conn->tcp_upload_params_virt_addr = p_tcp; | |
695 | ||
696 | r2tq_num_elements = p_params->num_r2tq_pages_in_ring * | |
697 | QED_CHAIN_PAGE_SIZE / 0x80; | |
698 | rc = qed_chain_alloc(p_hwfn->cdev, | |
699 | QED_CHAIN_USE_TO_CONSUME_PRODUCE, | |
700 | QED_CHAIN_MODE_PBL, | |
701 | QED_CHAIN_CNT_TYPE_U16, | |
702 | r2tq_num_elements, 0x80, &p_conn->r2tq); | |
703 | if (rc) | |
704 | goto nomem_r2tq; | |
705 | ||
706 | uhq_num_elements = p_params->num_uhq_pages_in_ring * | |
707 | QED_CHAIN_PAGE_SIZE / sizeof(struct iscsi_uhqe); | |
708 | rc = qed_chain_alloc(p_hwfn->cdev, | |
709 | QED_CHAIN_USE_TO_CONSUME_PRODUCE, | |
710 | QED_CHAIN_MODE_PBL, | |
711 | QED_CHAIN_CNT_TYPE_U16, | |
712 | uhq_num_elements, | |
713 | sizeof(struct iscsi_uhqe), &p_conn->uhq); | |
714 | if (rc) | |
715 | goto nomem_uhq; | |
716 | ||
717 | xhq_num_elements = uhq_num_elements; | |
718 | rc = qed_chain_alloc(p_hwfn->cdev, | |
719 | QED_CHAIN_USE_TO_CONSUME_PRODUCE, | |
720 | QED_CHAIN_MODE_PBL, | |
721 | QED_CHAIN_CNT_TYPE_U16, | |
722 | xhq_num_elements, | |
723 | sizeof(struct iscsi_xhqe), &p_conn->xhq); | |
724 | if (rc) | |
725 | goto nomem; | |
726 | ||
727 | p_conn->free_on_delete = true; | |
728 | *p_out_conn = p_conn; | |
729 | return 0; | |
730 | ||
731 | nomem: | |
732 | qed_chain_free(p_hwfn->cdev, &p_conn->uhq); | |
733 | nomem_uhq: | |
734 | qed_chain_free(p_hwfn->cdev, &p_conn->r2tq); | |
735 | nomem_r2tq: | |
736 | dma_free_coherent(&p_hwfn->cdev->pdev->dev, | |
737 | sizeof(struct tcp_upload_params), | |
738 | p_conn->tcp_upload_params_virt_addr, | |
739 | p_conn->tcp_upload_params_phys_addr); | |
740 | nomem_upload_param: | |
741 | dma_free_coherent(&p_hwfn->cdev->pdev->dev, | |
742 | sizeof(struct scsi_terminate_extra_params), | |
743 | p_conn->queue_cnts_virt_addr, | |
744 | p_conn->queue_cnts_phys_addr); | |
745 | nomem_queue_cnts_param: | |
746 | kfree(p_conn); | |
747 | ||
748 | return -ENOMEM; | |
749 | } | |
750 | ||
751 | static int qed_iscsi_acquire_connection(struct qed_hwfn *p_hwfn, | |
752 | struct qed_iscsi_conn *p_in_conn, | |
753 | struct qed_iscsi_conn **p_out_conn) | |
754 | { | |
755 | struct qed_iscsi_conn *p_conn = NULL; | |
756 | int rc = 0; | |
757 | u32 icid; | |
758 | ||
759 | spin_lock_bh(&p_hwfn->p_iscsi_info->lock); | |
760 | rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ISCSI, &icid); | |
761 | spin_unlock_bh(&p_hwfn->p_iscsi_info->lock); | |
762 | if (rc) | |
763 | return rc; | |
764 | ||
765 | /* Use input connection or allocate a new one */ | |
766 | if (p_in_conn) | |
767 | p_conn = p_in_conn; | |
768 | else | |
769 | rc = qed_iscsi_allocate_connection(p_hwfn, &p_conn); | |
770 | ||
771 | if (!rc) | |
772 | rc = qed_iscsi_setup_connection(p_hwfn, p_conn); | |
773 | ||
774 | if (rc) { | |
775 | spin_lock_bh(&p_hwfn->p_iscsi_info->lock); | |
776 | qed_cxt_release_cid(p_hwfn, icid); | |
777 | spin_unlock_bh(&p_hwfn->p_iscsi_info->lock); | |
778 | return rc; | |
779 | } | |
780 | ||
781 | p_conn->icid = icid; | |
782 | p_conn->conn_id = (u16)icid; | |
783 | p_conn->fw_cid = (p_hwfn->hw_info.opaque_fid << 16) | icid; | |
784 | ||
785 | *p_out_conn = p_conn; | |
786 | ||
787 | return rc; | |
788 | } | |
789 | ||
790 | static void qed_iscsi_release_connection(struct qed_hwfn *p_hwfn, | |
791 | struct qed_iscsi_conn *p_conn) | |
792 | { | |
793 | spin_lock_bh(&p_hwfn->p_iscsi_info->lock); | |
794 | list_add_tail(&p_conn->list_entry, &p_hwfn->p_iscsi_info->free_list); | |
795 | qed_cxt_release_cid(p_hwfn, p_conn->icid); | |
796 | spin_unlock_bh(&p_hwfn->p_iscsi_info->lock); | |
797 | } | |
798 | ||
4621ceb2 MY |
799 | void qed_iscsi_free_connection(struct qed_hwfn *p_hwfn, |
800 | struct qed_iscsi_conn *p_conn) | |
801 | { | |
802 | qed_chain_free(p_hwfn->cdev, &p_conn->xhq); | |
803 | qed_chain_free(p_hwfn->cdev, &p_conn->uhq); | |
804 | qed_chain_free(p_hwfn->cdev, &p_conn->r2tq); | |
805 | dma_free_coherent(&p_hwfn->cdev->pdev->dev, | |
806 | sizeof(struct tcp_upload_params), | |
807 | p_conn->tcp_upload_params_virt_addr, | |
808 | p_conn->tcp_upload_params_phys_addr); | |
809 | dma_free_coherent(&p_hwfn->cdev->pdev->dev, | |
810 | sizeof(struct scsi_terminate_extra_params), | |
811 | p_conn->queue_cnts_virt_addr, | |
812 | p_conn->queue_cnts_phys_addr); | |
813 | kfree(p_conn); | |
814 | } | |
815 | ||
fc831825 YM |
816 | struct qed_iscsi_info *qed_iscsi_alloc(struct qed_hwfn *p_hwfn) |
817 | { | |
818 | struct qed_iscsi_info *p_iscsi_info; | |
819 | ||
820 | p_iscsi_info = kzalloc(sizeof(*p_iscsi_info), GFP_KERNEL); | |
821 | if (!p_iscsi_info) | |
822 | return NULL; | |
823 | ||
824 | INIT_LIST_HEAD(&p_iscsi_info->free_list); | |
825 | return p_iscsi_info; | |
826 | } | |
827 | ||
828 | void qed_iscsi_setup(struct qed_hwfn *p_hwfn, | |
829 | struct qed_iscsi_info *p_iscsi_info) | |
830 | { | |
831 | spin_lock_init(&p_iscsi_info->lock); | |
832 | } | |
833 | ||
834 | void qed_iscsi_free(struct qed_hwfn *p_hwfn, | |
835 | struct qed_iscsi_info *p_iscsi_info) | |
836 | { | |
4621ceb2 MY |
837 | struct qed_iscsi_conn *p_conn = NULL; |
838 | ||
839 | while (!list_empty(&p_hwfn->p_iscsi_info->free_list)) { | |
840 | p_conn = list_first_entry(&p_hwfn->p_iscsi_info->free_list, | |
841 | struct qed_iscsi_conn, list_entry); | |
842 | if (p_conn) { | |
843 | list_del(&p_conn->list_entry); | |
844 | qed_iscsi_free_connection(p_hwfn, p_conn); | |
845 | } | |
846 | } | |
847 | ||
fc831825 YM |
848 | kfree(p_iscsi_info); |
849 | } | |
850 | ||
851 | static void _qed_iscsi_get_tstats(struct qed_hwfn *p_hwfn, | |
852 | struct qed_ptt *p_ptt, | |
853 | struct qed_iscsi_stats *p_stats) | |
854 | { | |
855 | struct tstorm_iscsi_stats_drv tstats; | |
856 | u32 tstats_addr; | |
857 | ||
858 | memset(&tstats, 0, sizeof(tstats)); | |
859 | tstats_addr = BAR0_MAP_REG_TSDM_RAM + | |
860 | TSTORM_ISCSI_RX_STATS_OFFSET(p_hwfn->rel_pf_id); | |
861 | qed_memcpy_from(p_hwfn, p_ptt, &tstats, tstats_addr, sizeof(tstats)); | |
862 | ||
863 | p_stats->iscsi_rx_bytes_cnt = | |
864 | HILO_64_REGPAIR(tstats.iscsi_rx_bytes_cnt); | |
865 | p_stats->iscsi_rx_packet_cnt = | |
866 | HILO_64_REGPAIR(tstats.iscsi_rx_packet_cnt); | |
867 | p_stats->iscsi_cmdq_threshold_cnt = | |
868 | le32_to_cpu(tstats.iscsi_cmdq_threshold_cnt); | |
869 | p_stats->iscsi_rq_threshold_cnt = | |
870 | le32_to_cpu(tstats.iscsi_rq_threshold_cnt); | |
871 | p_stats->iscsi_immq_threshold_cnt = | |
872 | le32_to_cpu(tstats.iscsi_immq_threshold_cnt); | |
873 | } | |
874 | ||
875 | static void _qed_iscsi_get_mstats(struct qed_hwfn *p_hwfn, | |
876 | struct qed_ptt *p_ptt, | |
877 | struct qed_iscsi_stats *p_stats) | |
878 | { | |
879 | struct mstorm_iscsi_stats_drv mstats; | |
880 | u32 mstats_addr; | |
881 | ||
882 | memset(&mstats, 0, sizeof(mstats)); | |
883 | mstats_addr = BAR0_MAP_REG_MSDM_RAM + | |
884 | MSTORM_ISCSI_RX_STATS_OFFSET(p_hwfn->rel_pf_id); | |
885 | qed_memcpy_from(p_hwfn, p_ptt, &mstats, mstats_addr, sizeof(mstats)); | |
886 | ||
887 | p_stats->iscsi_rx_dropped_pdus_task_not_valid = | |
888 | HILO_64_REGPAIR(mstats.iscsi_rx_dropped_pdus_task_not_valid); | |
889 | } | |
890 | ||
891 | static void _qed_iscsi_get_ustats(struct qed_hwfn *p_hwfn, | |
892 | struct qed_ptt *p_ptt, | |
893 | struct qed_iscsi_stats *p_stats) | |
894 | { | |
895 | struct ustorm_iscsi_stats_drv ustats; | |
896 | u32 ustats_addr; | |
897 | ||
898 | memset(&ustats, 0, sizeof(ustats)); | |
899 | ustats_addr = BAR0_MAP_REG_USDM_RAM + | |
900 | USTORM_ISCSI_RX_STATS_OFFSET(p_hwfn->rel_pf_id); | |
901 | qed_memcpy_from(p_hwfn, p_ptt, &ustats, ustats_addr, sizeof(ustats)); | |
902 | ||
903 | p_stats->iscsi_rx_data_pdu_cnt = | |
904 | HILO_64_REGPAIR(ustats.iscsi_rx_data_pdu_cnt); | |
905 | p_stats->iscsi_rx_r2t_pdu_cnt = | |
906 | HILO_64_REGPAIR(ustats.iscsi_rx_r2t_pdu_cnt); | |
907 | p_stats->iscsi_rx_total_pdu_cnt = | |
908 | HILO_64_REGPAIR(ustats.iscsi_rx_total_pdu_cnt); | |
909 | } | |
910 | ||
911 | static void _qed_iscsi_get_xstats(struct qed_hwfn *p_hwfn, | |
912 | struct qed_ptt *p_ptt, | |
913 | struct qed_iscsi_stats *p_stats) | |
914 | { | |
915 | struct xstorm_iscsi_stats_drv xstats; | |
916 | u32 xstats_addr; | |
917 | ||
918 | memset(&xstats, 0, sizeof(xstats)); | |
919 | xstats_addr = BAR0_MAP_REG_XSDM_RAM + | |
920 | XSTORM_ISCSI_TX_STATS_OFFSET(p_hwfn->rel_pf_id); | |
921 | qed_memcpy_from(p_hwfn, p_ptt, &xstats, xstats_addr, sizeof(xstats)); | |
922 | ||
923 | p_stats->iscsi_tx_go_to_slow_start_event_cnt = | |
924 | HILO_64_REGPAIR(xstats.iscsi_tx_go_to_slow_start_event_cnt); | |
925 | p_stats->iscsi_tx_fast_retransmit_event_cnt = | |
926 | HILO_64_REGPAIR(xstats.iscsi_tx_fast_retransmit_event_cnt); | |
927 | } | |
928 | ||
929 | static void _qed_iscsi_get_ystats(struct qed_hwfn *p_hwfn, | |
930 | struct qed_ptt *p_ptt, | |
931 | struct qed_iscsi_stats *p_stats) | |
932 | { | |
933 | struct ystorm_iscsi_stats_drv ystats; | |
934 | u32 ystats_addr; | |
935 | ||
936 | memset(&ystats, 0, sizeof(ystats)); | |
937 | ystats_addr = BAR0_MAP_REG_YSDM_RAM + | |
938 | YSTORM_ISCSI_TX_STATS_OFFSET(p_hwfn->rel_pf_id); | |
939 | qed_memcpy_from(p_hwfn, p_ptt, &ystats, ystats_addr, sizeof(ystats)); | |
940 | ||
941 | p_stats->iscsi_tx_data_pdu_cnt = | |
942 | HILO_64_REGPAIR(ystats.iscsi_tx_data_pdu_cnt); | |
943 | p_stats->iscsi_tx_r2t_pdu_cnt = | |
944 | HILO_64_REGPAIR(ystats.iscsi_tx_r2t_pdu_cnt); | |
945 | p_stats->iscsi_tx_total_pdu_cnt = | |
946 | HILO_64_REGPAIR(ystats.iscsi_tx_total_pdu_cnt); | |
947 | } | |
948 | ||
949 | static void _qed_iscsi_get_pstats(struct qed_hwfn *p_hwfn, | |
950 | struct qed_ptt *p_ptt, | |
951 | struct qed_iscsi_stats *p_stats) | |
952 | { | |
953 | struct pstorm_iscsi_stats_drv pstats; | |
954 | u32 pstats_addr; | |
955 | ||
956 | memset(&pstats, 0, sizeof(pstats)); | |
957 | pstats_addr = BAR0_MAP_REG_PSDM_RAM + | |
958 | PSTORM_ISCSI_TX_STATS_OFFSET(p_hwfn->rel_pf_id); | |
959 | qed_memcpy_from(p_hwfn, p_ptt, &pstats, pstats_addr, sizeof(pstats)); | |
960 | ||
961 | p_stats->iscsi_tx_bytes_cnt = | |
962 | HILO_64_REGPAIR(pstats.iscsi_tx_bytes_cnt); | |
963 | p_stats->iscsi_tx_packet_cnt = | |
964 | HILO_64_REGPAIR(pstats.iscsi_tx_packet_cnt); | |
965 | } | |
966 | ||
967 | static int qed_iscsi_get_stats(struct qed_hwfn *p_hwfn, | |
968 | struct qed_iscsi_stats *stats) | |
969 | { | |
970 | struct qed_ptt *p_ptt; | |
971 | ||
972 | memset(stats, 0, sizeof(*stats)); | |
973 | ||
974 | p_ptt = qed_ptt_acquire(p_hwfn); | |
975 | if (!p_ptt) { | |
976 | DP_ERR(p_hwfn, "Failed to acquire ptt\n"); | |
977 | return -EAGAIN; | |
978 | } | |
979 | ||
980 | _qed_iscsi_get_tstats(p_hwfn, p_ptt, stats); | |
981 | _qed_iscsi_get_mstats(p_hwfn, p_ptt, stats); | |
982 | _qed_iscsi_get_ustats(p_hwfn, p_ptt, stats); | |
983 | ||
984 | _qed_iscsi_get_xstats(p_hwfn, p_ptt, stats); | |
985 | _qed_iscsi_get_ystats(p_hwfn, p_ptt, stats); | |
986 | _qed_iscsi_get_pstats(p_hwfn, p_ptt, stats); | |
987 | ||
988 | qed_ptt_release(p_hwfn, p_ptt); | |
989 | ||
990 | return 0; | |
991 | } | |
992 | ||
993 | struct qed_hash_iscsi_con { | |
994 | struct hlist_node node; | |
995 | struct qed_iscsi_conn *con; | |
996 | }; | |
997 | ||
998 | static int qed_fill_iscsi_dev_info(struct qed_dev *cdev, | |
999 | struct qed_dev_iscsi_info *info) | |
1000 | { | |
1001 | struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); | |
1002 | ||
1003 | int rc; | |
1004 | ||
1005 | memset(info, 0, sizeof(*info)); | |
1006 | rc = qed_fill_dev_info(cdev, &info->common); | |
1007 | ||
1008 | info->primary_dbq_rq_addr = | |
1009 | qed_iscsi_get_primary_bdq_prod(hwfn, BDQ_ID_RQ); | |
1010 | info->secondary_bdq_rq_addr = | |
1011 | qed_iscsi_get_secondary_bdq_prod(hwfn, BDQ_ID_RQ); | |
1012 | ||
1013 | return rc; | |
1014 | } | |
1015 | ||
1016 | static void qed_register_iscsi_ops(struct qed_dev *cdev, | |
1017 | struct qed_iscsi_cb_ops *ops, void *cookie) | |
1018 | { | |
1019 | cdev->protocol_ops.iscsi = ops; | |
1020 | cdev->ops_cookie = cookie; | |
1021 | } | |
1022 | ||
1023 | static struct qed_hash_iscsi_con *qed_iscsi_get_hash(struct qed_dev *cdev, | |
1024 | u32 handle) | |
1025 | { | |
1026 | struct qed_hash_iscsi_con *hash_con = NULL; | |
1027 | ||
1028 | if (!(cdev->flags & QED_FLAG_STORAGE_STARTED)) | |
1029 | return NULL; | |
1030 | ||
1031 | hash_for_each_possible(cdev->connections, hash_con, node, handle) { | |
1032 | if (hash_con->con->icid == handle) | |
1033 | break; | |
1034 | } | |
1035 | ||
1036 | if (!hash_con || (hash_con->con->icid != handle)) | |
1037 | return NULL; | |
1038 | ||
1039 | return hash_con; | |
1040 | } | |
1041 | ||
1042 | static int qed_iscsi_stop(struct qed_dev *cdev) | |
1043 | { | |
1044 | int rc; | |
1045 | ||
1046 | if (!(cdev->flags & QED_FLAG_STORAGE_STARTED)) { | |
1047 | DP_NOTICE(cdev, "iscsi already stopped\n"); | |
1048 | return 0; | |
1049 | } | |
1050 | ||
1051 | if (!hash_empty(cdev->connections)) { | |
1052 | DP_NOTICE(cdev, | |
1053 | "Can't stop iscsi - not all connections were returned\n"); | |
1054 | return -EINVAL; | |
1055 | } | |
1056 | ||
1057 | /* Stop the iscsi */ | |
1058 | rc = qed_sp_iscsi_func_stop(QED_LEADING_HWFN(cdev), | |
1059 | QED_SPQ_MODE_EBLOCK, NULL); | |
1060 | cdev->flags &= ~QED_FLAG_STORAGE_STARTED; | |
1061 | ||
1062 | return rc; | |
1063 | } | |
1064 | ||
1065 | static int qed_iscsi_start(struct qed_dev *cdev, | |
1066 | struct qed_iscsi_tid *tasks, | |
1067 | void *event_context, | |
1068 | iscsi_event_cb_t async_event_cb) | |
1069 | { | |
1070 | int rc; | |
1071 | struct qed_tid_mem *tid_info; | |
1072 | ||
1073 | if (cdev->flags & QED_FLAG_STORAGE_STARTED) { | |
1074 | DP_NOTICE(cdev, "iscsi already started;\n"); | |
1075 | return 0; | |
1076 | } | |
1077 | ||
1078 | rc = qed_sp_iscsi_func_start(QED_LEADING_HWFN(cdev), | |
1079 | QED_SPQ_MODE_EBLOCK, NULL, event_context, | |
1080 | async_event_cb); | |
1081 | if (rc) { | |
1082 | DP_NOTICE(cdev, "Failed to start iscsi\n"); | |
1083 | return rc; | |
1084 | } | |
1085 | ||
1086 | cdev->flags |= QED_FLAG_STORAGE_STARTED; | |
1087 | hash_init(cdev->connections); | |
1088 | ||
1089 | if (!tasks) | |
1090 | return 0; | |
1091 | ||
1092 | tid_info = kzalloc(sizeof(*tid_info), GFP_KERNEL); | |
1093 | ||
1094 | if (!tid_info) { | |
1095 | qed_iscsi_stop(cdev); | |
1096 | return -ENOMEM; | |
1097 | } | |
1098 | ||
1099 | rc = qed_cxt_get_tid_mem_info(QED_LEADING_HWFN(cdev), | |
1100 | tid_info); | |
1101 | if (rc) { | |
1102 | DP_NOTICE(cdev, "Failed to gather task information\n"); | |
1103 | qed_iscsi_stop(cdev); | |
1104 | kfree(tid_info); | |
1105 | return rc; | |
1106 | } | |
1107 | ||
1108 | /* Fill task information */ | |
1109 | tasks->size = tid_info->tid_size; | |
1110 | tasks->num_tids_per_block = tid_info->num_tids_per_block; | |
1111 | memcpy(tasks->blocks, tid_info->blocks, | |
1112 | MAX_TID_BLOCKS_ISCSI * sizeof(u8 *)); | |
1113 | ||
1114 | kfree(tid_info); | |
1115 | ||
1116 | return 0; | |
1117 | } | |
1118 | ||
1119 | static int qed_iscsi_acquire_conn(struct qed_dev *cdev, | |
1120 | u32 *handle, | |
1121 | u32 *fw_cid, void __iomem **p_doorbell) | |
1122 | { | |
1123 | struct qed_hash_iscsi_con *hash_con; | |
1124 | int rc; | |
1125 | ||
1126 | /* Allocate a hashed connection */ | |
1127 | hash_con = kzalloc(sizeof(*hash_con), GFP_ATOMIC); | |
1128 | if (!hash_con) | |
1129 | return -ENOMEM; | |
1130 | ||
1131 | /* Acquire the connection */ | |
1132 | rc = qed_iscsi_acquire_connection(QED_LEADING_HWFN(cdev), NULL, | |
1133 | &hash_con->con); | |
1134 | if (rc) { | |
1135 | DP_NOTICE(cdev, "Failed to acquire Connection\n"); | |
1136 | kfree(hash_con); | |
1137 | return rc; | |
1138 | } | |
1139 | ||
1140 | /* Added the connection to hash table */ | |
1141 | *handle = hash_con->con->icid; | |
1142 | *fw_cid = hash_con->con->fw_cid; | |
1143 | hash_add(cdev->connections, &hash_con->node, *handle); | |
1144 | ||
1145 | if (p_doorbell) | |
1146 | *p_doorbell = qed_iscsi_get_db_addr(QED_LEADING_HWFN(cdev), | |
1147 | *handle); | |
1148 | ||
1149 | return 0; | |
1150 | } | |
1151 | ||
1152 | static int qed_iscsi_release_conn(struct qed_dev *cdev, u32 handle) | |
1153 | { | |
1154 | struct qed_hash_iscsi_con *hash_con; | |
1155 | ||
1156 | hash_con = qed_iscsi_get_hash(cdev, handle); | |
1157 | if (!hash_con) { | |
1158 | DP_NOTICE(cdev, "Failed to find connection for handle %d\n", | |
1159 | handle); | |
1160 | return -EINVAL; | |
1161 | } | |
1162 | ||
1163 | hlist_del(&hash_con->node); | |
1164 | qed_iscsi_release_connection(QED_LEADING_HWFN(cdev), hash_con->con); | |
1165 | kfree(hash_con); | |
1166 | ||
1167 | return 0; | |
1168 | } | |
1169 | ||
1170 | static int qed_iscsi_offload_conn(struct qed_dev *cdev, | |
1171 | u32 handle, | |
1172 | struct qed_iscsi_params_offload *conn_info) | |
1173 | { | |
1174 | struct qed_hash_iscsi_con *hash_con; | |
1175 | struct qed_iscsi_conn *con; | |
1176 | ||
1177 | hash_con = qed_iscsi_get_hash(cdev, handle); | |
1178 | if (!hash_con) { | |
1179 | DP_NOTICE(cdev, "Failed to find connection for handle %d\n", | |
1180 | handle); | |
1181 | return -EINVAL; | |
1182 | } | |
1183 | ||
1184 | /* Update the connection with information from the params */ | |
1185 | con = hash_con->con; | |
1186 | ||
1187 | ether_addr_copy(con->local_mac, conn_info->src.mac); | |
1188 | ether_addr_copy(con->remote_mac, conn_info->dst.mac); | |
1189 | memcpy(con->local_ip, conn_info->src.ip, sizeof(con->local_ip)); | |
1190 | memcpy(con->remote_ip, conn_info->dst.ip, sizeof(con->remote_ip)); | |
1191 | con->local_port = conn_info->src.port; | |
1192 | con->remote_port = conn_info->dst.port; | |
1193 | ||
1194 | con->layer_code = conn_info->layer_code; | |
1195 | con->sq_pbl_addr = conn_info->sq_pbl_addr; | |
1196 | con->initial_ack = conn_info->initial_ack; | |
1197 | con->vlan_id = conn_info->vlan_id; | |
1198 | con->tcp_flags = conn_info->tcp_flags; | |
1199 | con->ip_version = conn_info->ip_version; | |
1200 | con->default_cq = conn_info->default_cq; | |
1201 | con->ka_max_probe_cnt = conn_info->ka_max_probe_cnt; | |
1202 | con->dup_ack_theshold = conn_info->dup_ack_theshold; | |
1203 | con->rcv_next = conn_info->rcv_next; | |
1204 | con->snd_una = conn_info->snd_una; | |
1205 | con->snd_next = conn_info->snd_next; | |
1206 | con->snd_max = conn_info->snd_max; | |
1207 | con->snd_wnd = conn_info->snd_wnd; | |
1208 | con->rcv_wnd = conn_info->rcv_wnd; | |
1209 | con->snd_wl1 = conn_info->snd_wl1; | |
1210 | con->cwnd = conn_info->cwnd; | |
1211 | con->ss_thresh = conn_info->ss_thresh; | |
1212 | con->srtt = conn_info->srtt; | |
1213 | con->rtt_var = conn_info->rtt_var; | |
1214 | con->ts_time = conn_info->ts_time; | |
1215 | con->ts_recent = conn_info->ts_recent; | |
1216 | con->ts_recent_age = conn_info->ts_recent_age; | |
1217 | con->total_rt = conn_info->total_rt; | |
1218 | con->ka_timeout_delta = conn_info->ka_timeout_delta; | |
1219 | con->rt_timeout_delta = conn_info->rt_timeout_delta; | |
1220 | con->dup_ack_cnt = conn_info->dup_ack_cnt; | |
1221 | con->snd_wnd_probe_cnt = conn_info->snd_wnd_probe_cnt; | |
1222 | con->ka_probe_cnt = conn_info->ka_probe_cnt; | |
1223 | con->rt_cnt = conn_info->rt_cnt; | |
1224 | con->flow_label = conn_info->flow_label; | |
1225 | con->ka_timeout = conn_info->ka_timeout; | |
1226 | con->ka_interval = conn_info->ka_interval; | |
1227 | con->max_rt_time = conn_info->max_rt_time; | |
1228 | con->initial_rcv_wnd = conn_info->initial_rcv_wnd; | |
1229 | con->ttl = conn_info->ttl; | |
1230 | con->tos_or_tc = conn_info->tos_or_tc; | |
1231 | con->remote_port = conn_info->remote_port; | |
1232 | con->local_port = conn_info->local_port; | |
1233 | con->mss = conn_info->mss; | |
1234 | con->snd_wnd_scale = conn_info->snd_wnd_scale; | |
1235 | con->rcv_wnd_scale = conn_info->rcv_wnd_scale; | |
1236 | con->ts_ticks_per_second = conn_info->ts_ticks_per_second; | |
1237 | con->da_timeout_value = conn_info->da_timeout_value; | |
1238 | con->ack_frequency = conn_info->ack_frequency; | |
1239 | ||
1240 | /* Set default values on other connection fields */ | |
1241 | con->offl_flags = 0x1; | |
1242 | ||
1243 | return qed_sp_iscsi_conn_offload(QED_LEADING_HWFN(cdev), con, | |
1244 | QED_SPQ_MODE_EBLOCK, NULL); | |
1245 | } | |
1246 | ||
1247 | static int qed_iscsi_update_conn(struct qed_dev *cdev, | |
1248 | u32 handle, | |
1249 | struct qed_iscsi_params_update *conn_info) | |
1250 | { | |
1251 | struct qed_hash_iscsi_con *hash_con; | |
1252 | struct qed_iscsi_conn *con; | |
1253 | ||
1254 | hash_con = qed_iscsi_get_hash(cdev, handle); | |
1255 | if (!hash_con) { | |
1256 | DP_NOTICE(cdev, "Failed to find connection for handle %d\n", | |
1257 | handle); | |
1258 | return -EINVAL; | |
1259 | } | |
1260 | ||
1261 | /* Update the connection with information from the params */ | |
1262 | con = hash_con->con; | |
1263 | con->update_flag = conn_info->update_flag; | |
1264 | con->max_seq_size = conn_info->max_seq_size; | |
1265 | con->max_recv_pdu_length = conn_info->max_recv_pdu_length; | |
1266 | con->max_send_pdu_length = conn_info->max_send_pdu_length; | |
1267 | con->first_seq_length = conn_info->first_seq_length; | |
1268 | con->exp_stat_sn = conn_info->exp_stat_sn; | |
1269 | ||
1270 | return qed_sp_iscsi_conn_update(QED_LEADING_HWFN(cdev), con, | |
1271 | QED_SPQ_MODE_EBLOCK, NULL); | |
1272 | } | |
1273 | ||
1274 | static int qed_iscsi_clear_conn_sq(struct qed_dev *cdev, u32 handle) | |
1275 | { | |
1276 | struct qed_hash_iscsi_con *hash_con; | |
1277 | ||
1278 | hash_con = qed_iscsi_get_hash(cdev, handle); | |
1279 | if (!hash_con) { | |
1280 | DP_NOTICE(cdev, "Failed to find connection for handle %d\n", | |
1281 | handle); | |
1282 | return -EINVAL; | |
1283 | } | |
1284 | ||
1285 | return qed_sp_iscsi_conn_clear_sq(QED_LEADING_HWFN(cdev), | |
1286 | hash_con->con, | |
1287 | QED_SPQ_MODE_EBLOCK, NULL); | |
1288 | } | |
1289 | ||
1290 | static int qed_iscsi_destroy_conn(struct qed_dev *cdev, | |
1291 | u32 handle, u8 abrt_conn) | |
1292 | { | |
1293 | struct qed_hash_iscsi_con *hash_con; | |
1294 | ||
1295 | hash_con = qed_iscsi_get_hash(cdev, handle); | |
1296 | if (!hash_con) { | |
1297 | DP_NOTICE(cdev, "Failed to find connection for handle %d\n", | |
1298 | handle); | |
1299 | return -EINVAL; | |
1300 | } | |
1301 | ||
1302 | hash_con->con->abortive_dsconnect = abrt_conn; | |
1303 | ||
1304 | return qed_sp_iscsi_conn_terminate(QED_LEADING_HWFN(cdev), | |
1305 | hash_con->con, | |
1306 | QED_SPQ_MODE_EBLOCK, NULL); | |
1307 | } | |
1308 | ||
1309 | static int qed_iscsi_stats(struct qed_dev *cdev, struct qed_iscsi_stats *stats) | |
1310 | { | |
1311 | return qed_iscsi_get_stats(QED_LEADING_HWFN(cdev), stats); | |
1312 | } | |
1313 | ||
1314 | static const struct qed_iscsi_ops qed_iscsi_ops_pass = { | |
1315 | .common = &qed_common_ops_pass, | |
1316 | .ll2 = &qed_ll2_ops_pass, | |
1317 | .fill_dev_info = &qed_fill_iscsi_dev_info, | |
1318 | .register_ops = &qed_register_iscsi_ops, | |
1319 | .start = &qed_iscsi_start, | |
1320 | .stop = &qed_iscsi_stop, | |
1321 | .acquire_conn = &qed_iscsi_acquire_conn, | |
1322 | .release_conn = &qed_iscsi_release_conn, | |
1323 | .offload_conn = &qed_iscsi_offload_conn, | |
1324 | .update_conn = &qed_iscsi_update_conn, | |
1325 | .destroy_conn = &qed_iscsi_destroy_conn, | |
1326 | .clear_sq = &qed_iscsi_clear_conn_sq, | |
1327 | .get_stats = &qed_iscsi_stats, | |
1328 | }; | |
1329 | ||
7729bad4 | 1330 | const struct qed_iscsi_ops *qed_get_iscsi_ops(void) |
fc831825 YM |
1331 | { |
1332 | return &qed_iscsi_ops_pass; | |
1333 | } | |
1334 | EXPORT_SYMBOL(qed_get_iscsi_ops); | |
1335 | ||
1336 | void qed_put_iscsi_ops(void) | |
1337 | { | |
1338 | } | |
1339 | EXPORT_SYMBOL(qed_put_iscsi_ops); |