]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/scsi/csiostor/csio_wr.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/nab/target...
[mirror_ubuntu-artful-kernel.git] / drivers / scsi / csiostor / csio_wr.c
CommitLineData
a3667aae
NKI		 1/*
2 * This file is part of the Chelsio FCoE driver for Linux.
3 *
4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#include <linux/kernel.h>
36#include <linux/string.h>
37#include <linux/compiler.h>
38#include <linux/slab.h>
39#include <asm/page.h>
40#include <linux/cache.h>
41
42#include "csio_hw.h"
43#include "csio_wr.h"
44#include "csio_mb.h"
45#include "csio_defs.h"
46
47int csio_intr_coalesce_cnt; /* value:SGE_INGRESS_RX_THRESHOLD[0] */
48static int csio_sge_thresh_reg; /* SGE_INGRESS_RX_THRESHOLD[0] */
49
50int csio_intr_coalesce_time = 10; /* value:SGE_TIMER_VALUE_1 */
51static int csio_sge_timer_reg = 1;
52
53#define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val) \
f612b815 54 csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A)
a3667aae
NKI		 55
56static void
57csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg)
58{
f612b815 59 sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A +
a3667aae
NKI		 60 reg * sizeof(uint32_t));
61}
62
63/* Free list buffer size */
64static inline uint32_t
65csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf)
66{
67 return sge->sge_fl_buf_size[buf->paddr & 0xF];
68}
69
70/* Size of the egress queue status page */
71static inline uint32_t
72csio_wr_qstat_pgsz(struct csio_hw *hw)
73{
f612b815 74 return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64;
a3667aae
NKI		 75}
76
77/* Ring freelist doorbell */
78static inline void
79csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq)
80{
81 /*
82 * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ
83 * number of bytes in the freelist queue. This translates to atleast
84 * 8 freelist buffer pointers (since each pointer is 8 bytes).
85 */
86 if (flq->inc_idx >= 8) {
f612b815 87 csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) |
3fb4c22e 88 PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F,
f612b815 89 MYPF_REG(SGE_PF_KDOORBELL_A));
a3667aae
NKI		 90 flq->inc_idx &= 7;
91 }
92}
93
94/* Write a 0 cidx increment value to enable SGE interrupts for this queue */
95static void
96csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid)
97{
f612b815
HS		 98 csio_wr_reg32(hw, CIDXINC_V(0) |
99 INGRESSQID_V(iqid) |
100 TIMERREG_V(X_TIMERREG_RESTART_COUNTER),
101 MYPF_REG(SGE_PF_GTS_A));
a3667aae
NKI		 102}
103
104/*
105 * csio_wr_fill_fl - Populate the FL buffers of a FL queue.
106 * @hw: HW module.
107 * @flq: Freelist queue.
108 *
109 * Fill up freelist buffer entries with buffers of size specified
110 * in the size register.
111 *
112 */
113static int
114csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq)
115{
116 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
117 struct csio_sge *sge = &wrm->sge;
118 __be64 *d = (__be64 *)(flq->vstart);
119 struct csio_dma_buf *buf = &flq->un.fl.bufs[0];
120 uint64_t paddr;
121 int sreg = flq->un.fl.sreg;
122 int n = flq->credits;
123
124 while (n--) {
125 buf->len = sge->sge_fl_buf_size[sreg];
126 buf->vaddr = pci_alloc_consistent(hw->pdev, buf->len,
127 &buf->paddr);
128 if (!buf->vaddr) {
129 csio_err(hw, "Could only fill %d buffers!\n", n + 1);
130 return -ENOMEM;
131 }
132
133 paddr = buf->paddr | (sreg & 0xF);
134
135 *d++ = cpu_to_be64(paddr);
136 buf++;
137 }
138
139 return 0;
140}
141
142/*
143 * csio_wr_update_fl -
144 * @hw: HW module.
145 * @flq: Freelist queue.
146 *
147 *
148 */
149static inline void
150csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n)
151{
152
153 flq->inc_idx += n;
154 flq->pidx += n;
155 if (unlikely(flq->pidx >= flq->credits))
156 flq->pidx -= (uint16_t)flq->credits;
157
158 CSIO_INC_STATS(flq, n_flq_refill);
159}
160
161/*
162 * csio_wr_alloc_q - Allocate a WR queue and initialize it.
163 * @hw: HW module
164 * @qsize: Size of the queue in bytes
165 * @wrsize: Since of WR in this queue, if fixed.
166 * @type: Type of queue (Ingress/Egress/Freelist)
167 * @owner: Module that owns this queue.
168 * @nflb: Number of freelist buffers for FL.
169 * @sreg: What is the FL buffer size register?
170 * @iq_int_handler: Ingress queue handler in INTx mode.
171 *
172 * This function allocates and sets up a queue for the caller
173 * of size qsize, aligned at the required boundary. This is subject to
174 * be free entries being available in the queue array. If one is found,
175 * it is initialized with the allocated queue, marked as being used (owner),
176 * and a handle returned to the caller in form of the queue's index
177 * into the q_arr array.
178 * If user has indicated a freelist (by specifying nflb > 0), create
179 * another queue (with its own index into q_arr) for the freelist. Allocate
180 * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist
181 * idx in the ingress queue's flq.idx. This is how a Freelist is associated
182 * with its owning ingress queue.
183 */
184int
185csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize,
186 uint16_t type, void *owner, uint32_t nflb, int sreg,
187 iq_handler_t iq_intx_handler)
188{
189 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
190 struct csio_q *q, *flq;
191 int free_idx = wrm->free_qidx;
192 int ret_idx = free_idx;
193 uint32_t qsz;
194 int flq_idx;
195
196 if (free_idx >= wrm->num_q) {
197 csio_err(hw, "No more free queues.\n");
198 return -1;
199 }
200
201 switch (type) {
202 case CSIO_EGRESS:
203 qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw);
204 break;
205 case CSIO_INGRESS:
206 switch (wrsize) {
207 case 16:
208 case 32:
209 case 64:
210 case 128:
211 break;
212 default:
213 csio_err(hw, "Invalid Ingress queue WR size:%d\n",
214 wrsize);
215 return -1;
216 }
217
218 /*
219 * Number of elements must be a multiple of 16
220 * So this includes status page size
221 */
222 qsz = ALIGN(qsize/wrsize, 16) * wrsize;
223
224 break;
225 case CSIO_FREELIST:
226 qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw);
227 break;
228 default:
229 csio_err(hw, "Invalid queue type: 0x%x\n", type);
230 return -1;
231 }
232
233 q = wrm->q_arr[free_idx];
234
7c845eb5 235 q->vstart = pci_zalloc_consistent(hw->pdev, qsz, &q->pstart);
a3667aae
NKI		 236 if (!q->vstart) {
237 csio_err(hw,
238 "Failed to allocate DMA memory for "
239 "queue at id: %d size: %d\n", free_idx, qsize);
240 return -1;
241 }
242
a3667aae
NKI		 243 q->type = type;
244 q->owner = owner;
245 q->pidx = q->cidx = q->inc_idx = 0;
246 q->size = qsz;
247 q->wr_sz = wrsize; /* If using fixed size WRs */
248
249 wrm->free_qidx++;
250
251 if (type == CSIO_INGRESS) {
252 /* Since queue area is set to zero */
253 q->un.iq.genbit = 1;
254
255 /*
256 * Ingress queue status page size is always the size of
257 * the ingress queue entry.
258 */
259 q->credits = (qsz - q->wr_sz) / q->wr_sz;
260 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz
261 - q->wr_sz);
262
263 /* Allocate memory for FL if requested */
264 if (nflb > 0) {
265 flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64),
266 sizeof(__be64), CSIO_FREELIST,
267 owner, 0, sreg, NULL);
268 if (flq_idx == -1) {
269 csio_err(hw,
270 "Failed to allocate FL queue"
271 " for IQ idx:%d\n", free_idx);
272 return -1;
273 }
274
275 /* Associate the new FL with the Ingress quue */
276 q->un.iq.flq_idx = flq_idx;
277
278 flq = wrm->q_arr[q->un.iq.flq_idx];
279 flq->un.fl.bufs = kzalloc(flq->credits *
280 sizeof(struct csio_dma_buf),
281 GFP_KERNEL);
282 if (!flq->un.fl.bufs) {
283 csio_err(hw,
284 "Failed to allocate FL queue bufs"
285 " for IQ idx:%d\n", free_idx);
286 return -1;
287 }
288
289 flq->un.fl.packen = 0;
290 flq->un.fl.offset = 0;
291 flq->un.fl.sreg = sreg;
292
293 /* Fill up the free list buffers */
294 if (csio_wr_fill_fl(hw, flq))
295 return -1;
296
297 /*
298 * Make sure in a FLQ, atleast 1 credit (8 FL buffers)
299 * remains unpopulated,otherwise HW thinks
300 * FLQ is empty.
301 */
302 flq->pidx = flq->inc_idx = flq->credits - 8;
303 } else {
304 q->un.iq.flq_idx = -1;
305 }
306
307 /* Associate the IQ INTx handler. */
308 q->un.iq.iq_intx_handler = iq_intx_handler;
309
310 csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID;
311
312 } else if (type == CSIO_EGRESS) {
313 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ;
314 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz
315 - csio_wr_qstat_pgsz(hw));
316 csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID;
317 } else { /* Freelist */
318 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64);
319 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz
320 - csio_wr_qstat_pgsz(hw));
321 csio_q_flid(hw, ret_idx) = CSIO_MAX_QID;
322 }
323
324 return ret_idx;
325}
326
327/*
328 * csio_wr_iq_create_rsp - Response handler for IQ creation.
329 * @hw: The HW module.
330 * @mbp: Mailbox.
331 * @iq_idx: Ingress queue that got created.
332 *
333 * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids.
334 */
335static int
336csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
337{
338 struct csio_iq_params iqp;
339 enum fw_retval retval;
340 uint32_t iq_id;
341 int flq_idx;
342
343 memset(&iqp, 0, sizeof(struct csio_iq_params));
344
345 csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp);
346
347 if (retval != FW_SUCCESS) {
348 csio_err(hw, "IQ cmd returned 0x%x!\n", retval);
349 mempool_free(mbp, hw->mb_mempool);
350 return -EINVAL;
351 }
352
353 csio_q_iqid(hw, iq_idx) = iqp.iqid;
354 csio_q_physiqid(hw, iq_idx) = iqp.physiqid;
355 csio_q_pidx(hw, iq_idx) = csio_q_cidx(hw, iq_idx) = 0;
356 csio_q_inc_idx(hw, iq_idx) = 0;
357
358 /* Actual iq-id. */
359 iq_id = iqp.iqid - hw->wrm.fw_iq_start;
360
361 /* Set the iq-id to iq map table. */
362 if (iq_id >= CSIO_MAX_IQ) {
363 csio_err(hw,
364 "Exceeding MAX_IQ(%d) supported!"
365 " iqid:%d rel_iqid:%d FW iq_start:%d\n",
366 CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start);
367 mempool_free(mbp, hw->mb_mempool);
368 return -EINVAL;
369 }
370 csio_q_set_intr_map(hw, iq_idx, iq_id);
371
372 /*
373 * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE
374 * ingress context of this queue. This will block interrupts to
375 * this queue until the next GTS write. Therefore, we do a
376 * 0-cidx increment GTS write for this queue just to clear the
377 * interrupt_sent bit. This will re-enable interrupts to this
378 * queue.
379 */
380 csio_wr_sge_intr_enable(hw, iqp.physiqid);
381
382 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
383 if (flq_idx != -1) {
384 struct csio_q *flq = hw->wrm.q_arr[flq_idx];
385
386 csio_q_flid(hw, flq_idx) = iqp.fl0id;
387 csio_q_cidx(hw, flq_idx) = 0;
388 csio_q_pidx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;
389 csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;
390
391 /* Now update SGE about the buffers allocated during init */
392 csio_wr_ring_fldb(hw, flq);
393 }
394
395 mempool_free(mbp, hw->mb_mempool);
396
397 return 0;
398}
399
400/*
401 * csio_wr_iq_create - Configure an Ingress queue with FW.
402 * @hw: The HW module.
403 * @priv: Private data object.
404 * @iq_idx: Ingress queue index in the WR module.
405 * @vec: MSIX vector.
406 * @portid: PCIE Channel to be associated with this queue.
407 * @async: Is this a FW asynchronous message handling queue?
408 * @cbfn: Completion callback.
409 *
410 * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox
411 * with alloc/write bits set.
412 */
413int
414csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx,
415 uint32_t vec, uint8_t portid, bool async,
416 void (*cbfn) (struct csio_hw *, struct csio_mb *))
417{
418 struct csio_mb *mbp;
419 struct csio_iq_params iqp;
420 int flq_idx;
421
422 memset(&iqp, 0, sizeof(struct csio_iq_params));
423 csio_q_portid(hw, iq_idx) = portid;
424
425 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
426 if (!mbp) {
427 csio_err(hw, "IQ command out of memory!\n");
428 return -ENOMEM;
429 }
430
431 switch (hw->intr_mode) {
432 case CSIO_IM_INTX:
433 case CSIO_IM_MSI:
434 /* For interrupt forwarding queue only */
435 if (hw->intr_iq_idx == iq_idx)
436 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE;
437 else
438 iqp.iqandst = X_INTERRUPTDESTINATION_IQ;
439 iqp.iqandstindex =
440 csio_q_physiqid(hw, hw->intr_iq_idx);
441 break;
442 case CSIO_IM_MSIX:
443 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE;
444 iqp.iqandstindex = (uint16_t)vec;
445 break;
446 case CSIO_IM_NONE:
447 mempool_free(mbp, hw->mb_mempool);
448 return -EINVAL;
449 }
450
451 /* Pass in the ingress queue cmd parameters */
452 iqp.pfn = hw->pfn;
453 iqp.vfn = 0;
454 iqp.iq_start = 1;
455 iqp.viid = 0;
456 iqp.type = FW_IQ_TYPE_FL_INT_CAP;
457 iqp.iqasynch = async;
458 if (csio_intr_coalesce_cnt)
459 iqp.iqanus = X_UPDATESCHEDULING_COUNTER_OPTTIMER;
460 else
461 iqp.iqanus = X_UPDATESCHEDULING_TIMER;
462 iqp.iqanud = X_UPDATEDELIVERY_INTERRUPT;
463 iqp.iqpciech = portid;
464 iqp.iqintcntthresh = (uint8_t)csio_sge_thresh_reg;
465
466 switch (csio_q_wr_sz(hw, iq_idx)) {
467 case 16:
468 iqp.iqesize = 0; break;
469 case 32:
470 iqp.iqesize = 1; break;
471 case 64:
472 iqp.iqesize = 2; break;
473 case 128:
474 iqp.iqesize = 3; break;
475 }
476
477 iqp.iqsize = csio_q_size(hw, iq_idx) /
478 csio_q_wr_sz(hw, iq_idx);
479 iqp.iqaddr = csio_q_pstart(hw, iq_idx);
480
481 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
482 if (flq_idx != -1) {
483 struct csio_q *flq = hw->wrm.q_arr[flq_idx];
484
485 iqp.fl0paden = 1;
486 iqp.fl0packen = flq->un.fl.packen ? 1 : 0;
487 iqp.fl0fbmin = X_FETCHBURSTMIN_64B;
488 iqp.fl0fbmax = X_FETCHBURSTMAX_512B;
489 iqp.fl0size = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ;
490 iqp.fl0addr = csio_q_pstart(hw, flq_idx);
491 }
492
493 csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
494
495 if (csio_mb_issue(hw, mbp)) {
496 csio_err(hw, "Issue of IQ cmd failed!\n");
497 mempool_free(mbp, hw->mb_mempool);
498 return -EINVAL;
499 }
500
501 if (cbfn != NULL)
502 return 0;
503
504 return csio_wr_iq_create_rsp(hw, mbp, iq_idx);
505}
506
507/*
508 * csio_wr_eq_create_rsp - Response handler for EQ creation.
509 * @hw: The HW module.
510 * @mbp: Mailbox.
511 * @eq_idx: Egress queue that got created.
512 *
513 * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids.
514 */
515static int
516csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
517{
518 struct csio_eq_params eqp;
519 enum fw_retval retval;
520
521 memset(&eqp, 0, sizeof(struct csio_eq_params));
522
523 csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp);
524
525 if (retval != FW_SUCCESS) {
526 csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval);
527 mempool_free(mbp, hw->mb_mempool);
528 return -EINVAL;
529 }
530
531 csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid;
532 csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid;
533 csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0;
534 csio_q_inc_idx(hw, eq_idx) = 0;
535
536 mempool_free(mbp, hw->mb_mempool);
537
538 return 0;
539}
540
541/*
542 * csio_wr_eq_create - Configure an Egress queue with FW.
543 * @hw: HW module.
544 * @priv: Private data.
545 * @eq_idx: Egress queue index in the WR module.
546 * @iq_idx: Associated ingress queue index.
547 * @cbfn: Completion callback.
548 *
549 * This API configures a offload egress queue with FW by issuing a
550 * FW_EQ_OFLD_CMD (with alloc + write ) mailbox.
551 */
552int
553csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx,
554 int iq_idx, uint8_t portid,
555 void (*cbfn) (struct csio_hw *, struct csio_mb *))
556{
557 struct csio_mb *mbp;
558 struct csio_eq_params eqp;
559
560 memset(&eqp, 0, sizeof(struct csio_eq_params));
561
562 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
563 if (!mbp) {
564 csio_err(hw, "EQ command out of memory!\n");
565 return -ENOMEM;
566 }
567
568 eqp.pfn = hw->pfn;
569 eqp.vfn = 0;
570 eqp.eqstart = 1;
571 eqp.hostfcmode = X_HOSTFCMODE_STATUS_PAGE;
572 eqp.iqid = csio_q_iqid(hw, iq_idx);
573 eqp.fbmin = X_FETCHBURSTMIN_64B;
574 eqp.fbmax = X_FETCHBURSTMAX_512B;
575 eqp.cidxfthresh = 0;
576 eqp.pciechn = portid;
577 eqp.eqsize = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ;
578 eqp.eqaddr = csio_q_pstart(hw, eq_idx);
579
580 csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO,
581 &eqp, cbfn);
582
583 if (csio_mb_issue(hw, mbp)) {
584 csio_err(hw, "Issue of EQ OFLD cmd failed!\n");
585 mempool_free(mbp, hw->mb_mempool);
586 return -EINVAL;
587 }
588
589 if (cbfn != NULL)
590 return 0;
591
592 return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx);
593}
594
595/*
596 * csio_wr_iq_destroy_rsp - Response handler for IQ removal.
597 * @hw: The HW module.
598 * @mbp: Mailbox.
599 * @iq_idx: Ingress queue that was freed.
600 *
601 * Handle FW_IQ_CMD (free) mailbox completion.
602 */
603static int
604csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
605{
606 enum fw_retval retval = csio_mb_fw_retval(mbp);
607 int rv = 0;
608
609 if (retval != FW_SUCCESS)
610 rv = -EINVAL;
611
612 mempool_free(mbp, hw->mb_mempool);
613
614 return rv;
615}
616
617/*
618 * csio_wr_iq_destroy - Free an ingress queue.
619 * @hw: The HW module.
620 * @priv: Private data object.
621 * @iq_idx: Ingress queue index to destroy
622 * @cbfn: Completion callback.
623 *
624 * This API frees an ingress queue by issuing the FW_IQ_CMD
625 * with the free bit set.
626 */
627static int
628csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx,
629 void (*cbfn)(struct csio_hw *, struct csio_mb *))
630{
631 int rv = 0;
632 struct csio_mb *mbp;
633 struct csio_iq_params iqp;
634 int flq_idx;
635
636 memset(&iqp, 0, sizeof(struct csio_iq_params));
637
638 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
639 if (!mbp)
640 return -ENOMEM;
641
642 iqp.pfn = hw->pfn;
643 iqp.vfn = 0;
644 iqp.iqid = csio_q_iqid(hw, iq_idx);
645 iqp.type = FW_IQ_TYPE_FL_INT_CAP;
646
647 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
648 if (flq_idx != -1)
649 iqp.fl0id = csio_q_flid(hw, flq_idx);
650 else
651 iqp.fl0id = 0xFFFF;
652
653 iqp.fl1id = 0xFFFF;
654
655 csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
656
657 rv = csio_mb_issue(hw, mbp);
658 if (rv != 0) {
659 mempool_free(mbp, hw->mb_mempool);
660 return rv;
661 }
662
663 if (cbfn != NULL)
664 return 0;
665
666 return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx);
667}
668
669/*
670 * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation.
671 * @hw: The HW module.
672 * @mbp: Mailbox.
673 * @eq_idx: Egress queue that was freed.
674 *
675 * Handle FW_OFLD_EQ_CMD (free) mailbox completion.
676 */
677static int
678csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
679{
680 enum fw_retval retval = csio_mb_fw_retval(mbp);
681 int rv = 0;
682
683 if (retval != FW_SUCCESS)
684 rv = -EINVAL;
685
686 mempool_free(mbp, hw->mb_mempool);
687
688 return rv;
689}
690
691/*
692 * csio_wr_eq_destroy - Free an Egress queue.
693 * @hw: The HW module.
694 * @priv: Private data object.
695 * @eq_idx: Egress queue index to destroy
696 * @cbfn: Completion callback.
697 *
698 * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD
699 * with the free bit set.
700 */
701static int
702csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx,
703 void (*cbfn) (struct csio_hw *, struct csio_mb *))
704{
705 int rv = 0;
706 struct csio_mb *mbp;
707 struct csio_eq_params eqp;
708
709 memset(&eqp, 0, sizeof(struct csio_eq_params));
710
711 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
712 if (!mbp)
713 return -ENOMEM;
714
715 eqp.pfn = hw->pfn;
716 eqp.vfn = 0;
717 eqp.eqid = csio_q_eqid(hw, eq_idx);
718
719 csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn);
720
721 rv = csio_mb_issue(hw, mbp);
722 if (rv != 0) {
723 mempool_free(mbp, hw->mb_mempool);
724 return rv;
725 }
726
727 if (cbfn != NULL)
728 return 0;
729
730 return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx);
731}
732
733/*
734 * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page
735 * @hw: HW module
736 * @qidx: Egress queue index
737 *
738 * Cleanup the Egress queue status page.
739 */
740static void
741csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx)
742{
743 struct csio_q *q = csio_hw_to_wrm(hw)->q_arr[qidx];
744 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
745
746 memset(stp, 0, sizeof(*stp));
747}
748
749/*
750 * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ
751 * @hw: HW module
752 * @qidx: Ingress queue index
753 *
754 * Cleanup the footer entries in the given ingress queue,
755 * set to 1 the internal copy of genbit.
756 */
757static void
758csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx)
759{
760 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
761 struct csio_q *q = wrm->q_arr[qidx];
762 void *wr;
763 struct csio_iqwr_footer *ftr;
764 uint32_t i = 0;
765
766 /* set to 1 since we are just about zero out genbit */
767 q->un.iq.genbit = 1;
768
769 for (i = 0; i < q->credits; i++) {
770 /* Get the WR */
771 wr = (void *)((uintptr_t)q->vstart +
772 (i * q->wr_sz));
773 /* Get the footer */
774 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
775 (q->wr_sz - sizeof(*ftr)));
776 /* Zero out footer */
777 memset(ftr, 0, sizeof(*ftr));
778 }
779}
780
781int
782csio_wr_destroy_queues(struct csio_hw *hw, bool cmd)
783{
784 int i, flq_idx;
785 struct csio_q *q;
786 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
787 int rv;
788
789 for (i = 0; i < wrm->free_qidx; i++) {
790 q = wrm->q_arr[i];
791
792 switch (q->type) {
793 case CSIO_EGRESS:
794 if (csio_q_eqid(hw, i) != CSIO_MAX_QID) {
795 csio_wr_cleanup_eq_stpg(hw, i);
796 if (!cmd) {
797 csio_q_eqid(hw, i) = CSIO_MAX_QID;
798 continue;
799 }
800
801 rv = csio_wr_eq_destroy(hw, NULL, i, NULL);
802 if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
803 cmd = false;
804
805 csio_q_eqid(hw, i) = CSIO_MAX_QID;
806 }
807 case CSIO_INGRESS:
808 if (csio_q_iqid(hw, i) != CSIO_MAX_QID) {
809 csio_wr_cleanup_iq_ftr(hw, i);
810 if (!cmd) {
811 csio_q_iqid(hw, i) = CSIO_MAX_QID;
812 flq_idx = csio_q_iq_flq_idx(hw, i);
813 if (flq_idx != -1)
814 csio_q_flid(hw, flq_idx) =
815 CSIO_MAX_QID;
816 continue;
817 }
818
819 rv = csio_wr_iq_destroy(hw, NULL, i, NULL);
820 if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
821 cmd = false;
822
823 csio_q_iqid(hw, i) = CSIO_MAX_QID;
824 flq_idx = csio_q_iq_flq_idx(hw, i);
825 if (flq_idx != -1)
826 csio_q_flid(hw, flq_idx) = CSIO_MAX_QID;
827 }
828 default:
829 break;
830 }
831 }
832
833 hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED;
834
835 return 0;
836}
837
838/*
839 * csio_wr_get - Get requested size of WR entry/entries from queue.
840 * @hw: HW module.
841 * @qidx: Index of queue.
842 * @size: Cumulative size of Work request(s).
843 * @wrp: Work request pair.
844 *
845 * If requested credits are available, return the start address of the
846 * work request in the work request pair. Set pidx accordingly and
847 * return.
848 *
849 * NOTE about WR pair:
850 * ==================
851 * A WR can start towards the end of a queue, and then continue at the
852 * beginning, since the queue is considered to be circular. This will
853 * require a pair of address/size to be passed back to the caller -
854 * hence Work request pair format.
855 */
856int
857csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size,
858 struct csio_wr_pair *wrp)
859{
860 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
861 struct csio_q *q = wrm->q_arr[qidx];
862 void *cwr = (void *)((uintptr_t)(q->vstart) +
863 (q->pidx * CSIO_QCREDIT_SZ));
864 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
865 uint16_t cidx = q->cidx = ntohs(stp->cidx);
866 uint16_t pidx = q->pidx;
867 uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ);
868 int req_credits = req_sz / CSIO_QCREDIT_SZ;
869 int credits;
870
871 CSIO_DB_ASSERT(q->owner != NULL);
872 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
873 CSIO_DB_ASSERT(cidx <= q->credits);
874
875 /* Calculate credits */
876 if (pidx > cidx) {
877 credits = q->credits - (pidx - cidx) - 1;
878 } else if (cidx > pidx) {
879 credits = cidx - pidx - 1;
880 } else {
881 /* cidx == pidx, empty queue */
882 credits = q->credits;
883 CSIO_INC_STATS(q, n_qempty);
884 }
885
886 /*
887 * Check if we have enough credits.
888 * credits = 1 implies queue is full.
889 */
890 if (!credits || (req_credits > credits)) {
891 CSIO_INC_STATS(q, n_qfull);
892 return -EBUSY;
893 }
894
895 /*
896 * If we are here, we have enough credits to satisfy the
897 * request. Check if we are near the end of q, and if WR spills over.
898 * If it does, use the first addr/size to cover the queue until
899 * the end. Fit the remainder portion of the request at the top
900 * of queue and return it in the second addr/len. Set pidx
901 * accordingly.
902 */
903 if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) {
904 wrp->addr1 = cwr;
905 wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr);
906 wrp->addr2 = q->vstart;
907 wrp->size2 = req_sz - wrp->size1;
908 q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) /
909 CSIO_QCREDIT_SZ);
910 CSIO_INC_STATS(q, n_qwrap);
911 CSIO_INC_STATS(q, n_eq_wr_split);
912 } else {
913 wrp->addr1 = cwr;
914 wrp->size1 = req_sz;
915 wrp->addr2 = NULL;
916 wrp->size2 = 0;
917 q->pidx += (uint16_t)req_credits;
918
919 /* We are the end of queue, roll back pidx to top of queue */
920 if (unlikely(q->pidx == q->credits)) {
921 q->pidx = 0;
922 CSIO_INC_STATS(q, n_qwrap);
923 }
924 }
925
926 q->inc_idx = (uint16_t)req_credits;
927
928 CSIO_INC_STATS(q, n_tot_reqs);
929
930 return 0;
931}
932
933/*
934 * csio_wr_copy_to_wrp - Copies given data into WR.
935 * @data_buf - Data buffer
936 * @wrp - Work request pair.
937 * @wr_off - Work request offset.
938 * @data_len - Data length.
939 *
940 * Copies the given data in Work Request. Work request pair(wrp) specifies
941 * address information of Work request.
942 * Returns: none
943 */
944void
945csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp,
946 uint32_t wr_off, uint32_t data_len)
947{
948 uint32_t nbytes;
949
950 /* Number of space available in buffer addr1 of WRP */
951 nbytes = ((wrp->size1 - wr_off) >= data_len) ?
952 data_len : (wrp->size1 - wr_off);
953
954 memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes);
955 data_len -= nbytes;
956
957 /* Write the remaining data from the begining of circular buffer */
958 if (data_len) {
959 CSIO_DB_ASSERT(data_len <= wrp->size2);
960 CSIO_DB_ASSERT(wrp->addr2 != NULL);
961 memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len);
962 }
963}
964
965/*
966 * csio_wr_issue - Notify chip of Work request.
967 * @hw: HW module.
968 * @qidx: Index of queue.
969 * @prio: 0: Low priority, 1: High priority
970 *
971 * Rings the SGE Doorbell by writing the current producer index of the passed
972 * in queue into the register.
973 *
974 */
975int
976csio_wr_issue(struct csio_hw *hw, int qidx, bool prio)
977{
978 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
979 struct csio_q *q = wrm->q_arr[qidx];
980
981 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
982
983 wmb();
984 /* Ring SGE Doorbell writing q->pidx into it */
f612b815 985 csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) |
3fb4c22e 986 PIDX_T5_V(q->inc_idx) | DBTYPE_F,
f612b815 987 MYPF_REG(SGE_PF_KDOORBELL_A));
a3667aae
NKI		 988 q->inc_idx = 0;
989
990 return 0;
991}
992
993static inline uint32_t
994csio_wr_avail_qcredits(struct csio_q *q)
995{
996 if (q->pidx > q->cidx)
997 return q->pidx - q->cidx;
998 else if (q->cidx > q->pidx)
999 return q->credits - (q->cidx - q->pidx);
1000 else
1001 return 0; /* cidx == pidx, empty queue */
1002}
1003
1004/*
1005 * csio_wr_inval_flq_buf - Invalidate a free list buffer entry.
1006 * @hw: HW module.
1007 * @flq: The freelist queue.
1008 *
1009 * Invalidate the driver's version of a freelist buffer entry,
1010 * without freeing the associated the DMA memory. The entry
1011 * to be invalidated is picked up from the current Free list
1012 * queue cidx.
1013 *
1014 */
1015static inline void
1016csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq)
1017{
1018 flq->cidx++;
1019 if (flq->cidx == flq->credits) {
1020 flq->cidx = 0;
1021 CSIO_INC_STATS(flq, n_qwrap);
1022 }
1023}
1024
1025/*
1026 * csio_wr_process_fl - Process a freelist completion.
1027 * @hw: HW module.
1028 * @q: The ingress queue attached to the Freelist.
1029 * @wr: The freelist completion WR in the ingress queue.
1030 * @len_to_qid: The lower 32-bits of the first flit of the RSP footer
1031 * @iq_handler: Caller's handler for this completion.
1032 * @priv: Private pointer of caller
1033 *
1034 */
1035static inline void
1036csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q,
1037 void *wr, uint32_t len_to_qid,
1038 void (*iq_handler)(struct csio_hw *, void *,
1039 uint32_t, struct csio_fl_dma_buf *,
1040 void *),
1041 void *priv)
1042{
1043 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1044 struct csio_sge *sge = &wrm->sge;
1045 struct csio_fl_dma_buf flb;
1046 struct csio_dma_buf *buf, *fbuf;
1047 uint32_t bufsz, len, lastlen = 0;
1048 int n;
1049 struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx];
1050
1051 CSIO_DB_ASSERT(flq != NULL);
1052
1053 len = len_to_qid;
1054
1055 if (len & IQWRF_NEWBUF) {
1056 if (flq->un.fl.offset > 0) {
1057 csio_wr_inval_flq_buf(hw, flq);
1058 flq->un.fl.offset = 0;
1059 }
1060 len = IQWRF_LEN_GET(len);
1061 }
1062
1063 CSIO_DB_ASSERT(len != 0);
1064
1065 flb.totlen = len;
1066
1067 /* Consume all freelist buffers used for len bytes */
1068 for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) {
1069 buf = &flq->un.fl.bufs[flq->cidx];
1070 bufsz = csio_wr_fl_bufsz(sge, buf);
1071
1072 fbuf->paddr = buf->paddr;
1073 fbuf->vaddr = buf->vaddr;
1074
1075 flb.offset = flq->un.fl.offset;
1076 lastlen = min(bufsz, len);
1077 fbuf->len = lastlen;
1078
1079 len -= lastlen;
1080 if (!len)
1081 break;
1082 csio_wr_inval_flq_buf(hw, flq);
1083 }
1084
1085 flb.defer_free = flq->un.fl.packen ? 0 : 1;
1086
1087 iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer),
1088 &flb, priv);
1089
1090 if (flq->un.fl.packen)
1091 flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align);
1092 else
1093 csio_wr_inval_flq_buf(hw, flq);
1094
1095}
1096
1097/*
1098 * csio_is_new_iqwr - Is this a new Ingress queue entry ?
1099 * @q: Ingress quueue.
1100 * @ftr: Ingress queue WR SGE footer.
1101 *
1102 * The entry is new if our generation bit matches the corresponding
1103 * bit in the footer of the current WR.
1104 */
1105static inline bool
1106csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr)
1107{
1108 return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT));
1109}
1110
1111/*
1112 * csio_wr_process_iq - Process elements in Ingress queue.
1113 * @hw: HW pointer
1114 * @qidx: Index of queue
1115 * @iq_handler: Handler for this queue
1116 * @priv: Caller's private pointer
1117 *
1118 * This routine walks through every entry of the ingress queue, calling
1119 * the provided iq_handler with the entry, until the generation bit
1120 * flips.
1121 */
1122int
1123csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q,
1124 void (*iq_handler)(struct csio_hw *, void *,
1125 uint32_t, struct csio_fl_dma_buf *,
1126 void *),
1127 void *priv)
1128{
1129 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1130 void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz));
1131 struct csio_iqwr_footer *ftr;
1132 uint32_t wr_type, fw_qid, qid;
1133 struct csio_q *q_completed;
1134 struct csio_q *flq = csio_iq_has_fl(q) ?
1135 wrm->q_arr[q->un.iq.flq_idx] : NULL;
1136 int rv = 0;
1137
1138 /* Get the footer */
1139 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1140 (q->wr_sz - sizeof(*ftr)));
1141
1142 /*
1143 * When q wrapped around last time, driver should have inverted
1144 * ic.genbit as well.
1145 */
1146 while (csio_is_new_iqwr(q, ftr)) {
1147
1148 CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <=
1149 (uintptr_t)q->vwrap);
1150 rmb();
1151 wr_type = IQWRF_TYPE_GET(ftr->u.type_gen);
1152
1153 switch (wr_type) {
1154 case X_RSPD_TYPE_CPL:
1155 /* Subtract footer from WR len */
1156 iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv);
1157 break;
1158 case X_RSPD_TYPE_FLBUF:
1159 csio_wr_process_fl(hw, q, wr,
1160 ntohl(ftr->pldbuflen_qid),
1161 iq_handler, priv);
1162 break;
1163 case X_RSPD_TYPE_INTR:
1164 fw_qid = ntohl(ftr->pldbuflen_qid);
1165 qid = fw_qid - wrm->fw_iq_start;
1166 q_completed = hw->wrm.intr_map[qid];
1167
1168 if (unlikely(qid ==
1169 csio_q_physiqid(hw, hw->intr_iq_idx))) {
1170 /*
1171 * We are already in the Forward Interrupt
1172 * Interrupt Queue Service! Do-not service
1173 * again!
1174 *
1175 */
1176 } else {
1177 CSIO_DB_ASSERT(q_completed);
1178 CSIO_DB_ASSERT(
1179 q_completed->un.iq.iq_intx_handler);
1180
1181 /* Call the queue handler. */
1182 q_completed->un.iq.iq_intx_handler(hw, NULL,
1183 0, NULL, (void *)q_completed);
1184 }
1185 break;
1186 default:
1187 csio_warn(hw, "Unknown resp type 0x%x received\n",
1188 wr_type);
1189 CSIO_INC_STATS(q, n_rsp_unknown);
1190 break;
1191 }
1192
1193 /*
1194 * Ingress *always* has fixed size WR entries. Therefore,
1195 * there should always be complete WRs towards the end of
1196 * queue.
1197 */
1198 if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) {
1199
1200 /* Roll over to start of queue */
1201 q->cidx = 0;
1202 wr = q->vstart;
1203
1204 /* Toggle genbit */
1205 q->un.iq.genbit ^= 0x1;
1206
1207 CSIO_INC_STATS(q, n_qwrap);
1208 } else {
1209 q->cidx++;
1210 wr = (void *)((uintptr_t)(q->vstart) +
1211 (q->cidx * q->wr_sz));
1212 }
1213
1214 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1215 (q->wr_sz - sizeof(*ftr)));
1216 q->inc_idx++;
1217
1218 } /* while (q->un.iq.genbit == hdr->genbit) */
1219
1220 /*
1221 * We need to re-arm SGE interrupts in case we got a stray interrupt,
1222 * especially in msix mode. With INTx, this may be a common occurence.
1223 */
1224 if (unlikely(!q->inc_idx)) {
1225 CSIO_INC_STATS(q, n_stray_comp);
1226 rv = -EINVAL;
1227 goto restart;
1228 }
1229
1230 /* Replenish free list buffers if pending falls below low water mark */
1231 if (flq) {
1232 uint32_t avail = csio_wr_avail_qcredits(flq);
1233 if (avail <= 16) {
1234 /* Make sure in FLQ, atleast 1 credit (8 FL buffers)
1235 * remains unpopulated otherwise HW thinks
1236 * FLQ is empty.
1237 */
1238 csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail);
1239 csio_wr_ring_fldb(hw, flq);
1240 }
1241 }
1242
1243restart:
1244 /* Now inform SGE about our incremental index value */
f612b815
HS		 1245 csio_wr_reg32(hw, CIDXINC_V(q->inc_idx) |
1246 INGRESSQID_V(q->un.iq.physiqid) |
1247 TIMERREG_V(csio_sge_timer_reg),
1248 MYPF_REG(SGE_PF_GTS_A));
a3667aae
NKI		 1249 q->stats.n_tot_rsps += q->inc_idx;
1250
1251 q->inc_idx = 0;
1252
1253 return rv;
1254}
1255
1256int
1257csio_wr_process_iq_idx(struct csio_hw *hw, int qidx,
1258 void (*iq_handler)(struct csio_hw *, void *,
1259 uint32_t, struct csio_fl_dma_buf *,
1260 void *),
1261 void *priv)
1262{
1263 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1264 struct csio_q *iq = wrm->q_arr[qidx];
1265
1266 return csio_wr_process_iq(hw, iq, iq_handler, priv);
1267}
1268
1269static int
1270csio_closest_timer(struct csio_sge *s, int time)
1271{
1272 int i, delta, match = 0, min_delta = INT_MAX;
1273
1274 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1275 delta = time - s->timer_val[i];
1276 if (delta < 0)
1277 delta = -delta;
1278 if (delta < min_delta) {
1279 min_delta = delta;
1280 match = i;
1281 }
1282 }
1283 return match;
1284}
1285
1286static int
1287csio_closest_thresh(struct csio_sge *s, int cnt)
1288{
1289 int i, delta, match = 0, min_delta = INT_MAX;
1290
1291 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1292 delta = cnt - s->counter_val[i];
1293 if (delta < 0)
1294 delta = -delta;
1295 if (delta < min_delta) {
1296 min_delta = delta;
1297 match = i;
1298 }
1299 }
1300 return match;
1301}
1302
1303static void
1304csio_wr_fixup_host_params(struct csio_hw *hw)
1305{
1306 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1307 struct csio_sge *sge = &wrm->sge;
1308 uint32_t clsz = L1_CACHE_BYTES;
1309 uint32_t s_hps = PAGE_SHIFT - 10;
1310 uint32_t ingpad = 0;
1311 uint32_t stat_len = clsz > 64 ? 128 : 64;
1312
f612b815
HS		 1313 csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) |
1314 HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) |
1315 HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) |
1316 HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps),
1317 SGE_HOST_PAGE_SIZE_A);
a3667aae
NKI		 1318
1319 sge->csio_fl_align = clsz < 32 ? 32 : clsz;
1320 ingpad = ilog2(sge->csio_fl_align) - 5;
1321
f612b815
HS		 1322 csio_set_reg_field(hw, SGE_CONTROL_A,
1323 INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
1324 EGRSTATUSPAGESIZE_F,
1325 INGPADBOUNDARY_V(ingpad) |
1326 EGRSTATUSPAGESIZE_V(stat_len != 64));
a3667aae
NKI		 1327
1328 /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */
f612b815 1329 csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A);
d69630e8
AB		 1330
1331 /*
1332 * If using hard params, the following will get set correctly
1333 * in csio_wr_set_sge().
1334 */
1335 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) {
1336 csio_wr_reg32(hw,
f612b815 1337 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) +
d69630e8 1338 sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1),
f612b815 1339 SGE_FL_BUFFER_SIZE2_A);
d69630e8 1340 csio_wr_reg32(hw,
f612b815 1341 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) +
d69630e8 1342 sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1),
f612b815 1343 SGE_FL_BUFFER_SIZE3_A);
d69630e8 1344 }
a3667aae 1345
0d804338 1346 csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A);
a3667aae
NKI		 1347
1348 /* default value of rx_dma_offset of the NIC driver */
f612b815
HS		 1349 csio_set_reg_field(hw, SGE_CONTROL_A,
1350 PKTSHIFT_V(PKTSHIFT_M),
1351 PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET));
7cc16380 1352
837e4a42
HS		 1353 csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A,
1354 CSUM_HAS_PSEUDO_HDR_F, 0);
a3667aae
NKI		 1355}
1356
1357static void
1358csio_init_intr_coalesce_parms(struct csio_hw *hw)
1359{
1360 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1361 struct csio_sge *sge = &wrm->sge;
1362
1363 csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt);
1364 if (csio_intr_coalesce_cnt) {
1365 csio_sge_thresh_reg = 0;
1366 csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER;
1367 return;
1368 }
1369
1370 csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time);
1371}
1372
1373/*
1374 * csio_wr_get_sge - Get SGE register values.
1375 * @hw: HW module.
1376 *
1377 * Used by non-master functions and by master-functions relying on config file.
1378 */
1379static void
1380csio_wr_get_sge(struct csio_hw *hw)
1381{
1382 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1383 struct csio_sge *sge = &wrm->sge;
1384 uint32_t ingpad;
1385 int i;
1386 u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
1387 u32 ingress_rx_threshold;
1388
f612b815 1389 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);
a3667aae 1390
f612b815 1391 ingpad = INGPADBOUNDARY_G(sge->sge_control);
a3667aae
NKI		 1392
1393 switch (ingpad) {
1394 case X_INGPCIEBOUNDARY_32B:
1395 sge->csio_fl_align = 32; break;
1396 case X_INGPCIEBOUNDARY_64B:
1397 sge->csio_fl_align = 64; break;
1398 case X_INGPCIEBOUNDARY_128B:
1399 sge->csio_fl_align = 128; break;
1400 case X_INGPCIEBOUNDARY_256B:
1401 sge->csio_fl_align = 256; break;
1402 case X_INGPCIEBOUNDARY_512B:
1403 sge->csio_fl_align = 512; break;
1404 case X_INGPCIEBOUNDARY_1024B:
1405 sge->csio_fl_align = 1024; break;
1406 case X_INGPCIEBOUNDARY_2048B:
1407 sge->csio_fl_align = 2048; break;
1408 case X_INGPCIEBOUNDARY_4096B:
1409 sge->csio_fl_align = 4096; break;
1410 }
1411
1412 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1413 csio_get_flbuf_size(hw, sge, i);
1414
f061de42
HS		 1415 timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A);
1416 timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A);
1417 timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A);
a3667aae
NKI		 1418
1419 sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw,
f061de42 1420 TIMERVALUE0_G(timer_value_0_and_1));
a3667aae 1421 sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw,
f061de42 1422 TIMERVALUE1_G(timer_value_0_and_1));
a3667aae 1423 sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw,
f061de42 1424 TIMERVALUE2_G(timer_value_2_and_3));
a3667aae 1425 sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw,
f061de42 1426 TIMERVALUE3_G(timer_value_2_and_3));
a3667aae 1427 sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw,
f061de42 1428 TIMERVALUE4_G(timer_value_4_and_5));
a3667aae 1429 sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw,
f061de42 1430 TIMERVALUE5_G(timer_value_4_and_5));
a3667aae 1431
f612b815
HS		 1432 ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A);
1433 sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
1434 sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
1435 sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
1436 sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);
a3667aae
NKI		 1437
1438 csio_init_intr_coalesce_parms(hw);
1439}
1440
1441/*
1442 * csio_wr_set_sge - Initialize SGE registers
1443 * @hw: HW module.
1444 *
1445 * Used by Master function to initialize SGE registers in the absence
1446 * of a config file.
1447 */
1448static void
1449csio_wr_set_sge(struct csio_hw *hw)
1450{
1451 struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1452 struct csio_sge *sge = &wrm->sge;
1453 int i;
1454
1455 /*
1456 * Set up our basic SGE mode to deliver CPL messages to our Ingress
1457 * Queue and Packet Date to the Free List.
1458 */
f612b815 1459 csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F);
a3667aae 1460
f612b815 1461 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);
a3667aae
NKI		 1462
1463 /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */
1464
1465 /*
1466 * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows
1467 * and generate an interrupt when this occurs so we can recover.
1468 */
f612b815 1469 csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A,
3fb4c22e
PM		 1470 LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
1471 LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
1472 csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A,
1473 HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
1474 HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
7cc16380 1475
f612b815
HS		 1476 csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F,
1477 ENABLE_DROP_F);
a3667aae
NKI		 1478
1479 /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */
1480
1481 CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1);
d69630e8 1482 csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1)
f612b815 1483 & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A);
d69630e8 1484 csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1)
f612b815 1485 & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A);
a3667aae
NKI		 1486 CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4);
1487 CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5);
1488 CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6);
1489 CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7);
1490 CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8);
1491
1492 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1493 csio_get_flbuf_size(hw, sge, i);
1494
1495 /* Initialize interrupt coalescing attributes */
1496 sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0;
1497 sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1;
1498 sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2;
1499 sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3;
1500 sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4;
1501 sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5;
1502
1503 sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0;
1504 sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1;
1505 sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2;
1506 sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3;
1507
f612b815
HS		 1508 csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) |
1509 THRESHOLD_1_V(sge->counter_val[1]) |
1510 THRESHOLD_2_V(sge->counter_val[2]) |
1511 THRESHOLD_3_V(sge->counter_val[3]),
1512 SGE_INGRESS_RX_THRESHOLD_A);
a3667aae
NKI		 1513
1514 csio_wr_reg32(hw,
f061de42
HS		 1515 TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) |
1516 TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])),
1517 SGE_TIMER_VALUE_0_AND_1_A);
a3667aae
NKI		 1518
1519 csio_wr_reg32(hw,
f061de42
HS		 1520 TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) |
1521 TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])),
1522 SGE_TIMER_VALUE_2_AND_3_A);
a3667aae
NKI		 1523
1524 csio_wr_reg32(hw,
f061de42
HS		 1525 TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) |
1526 TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])),
1527 SGE_TIMER_VALUE_4_AND_5_A);
a3667aae
NKI		 1528
1529 csio_init_intr_coalesce_parms(hw);
1530}
1531
1532void
1533csio_wr_sge_init(struct csio_hw *hw)
1534{
1535 /*
d69630e8 1536 * If we are master and chip is not initialized:
a3667aae
NKI		 1537 * - If we plan to use the config file, we need to fixup some
1538 * host specific registers, and read the rest of the SGE
1539 * configuration.
1540 * - If we dont plan to use the config file, we need to initialize
1541 * SGE entirely, including fixing the host specific registers.
d69630e8
AB		 1542 * If we are master and chip is initialized, just read and work off of
1543 * the already initialized SGE values.
a3667aae
NKI		 1544 * If we arent the master, we are only allowed to read and work off of
1545 * the already initialized SGE values.
1546 *
1547 * Therefore, before calling this function, we assume that the master-
d69630e8
AB		 1548 * ship of the card, state and whether to use config file or not, have
1549 * already been decided.
a3667aae
NKI		 1550 */
1551 if (csio_is_hw_master(hw)) {
d69630e8
AB		 1552 if (hw->fw_state != CSIO_DEV_STATE_INIT)
1553 csio_wr_fixup_host_params(hw);
a3667aae
NKI		 1554
1555 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS)
1556 csio_wr_get_sge(hw);
1557 else
1558 csio_wr_set_sge(hw);
1559 } else
1560 csio_wr_get_sge(hw);
1561}
1562
1563/*
1564 * csio_wrm_init - Initialize Work request module.
1565 * @wrm: WR module
1566 * @hw: HW pointer
1567 *
1568 * Allocates memory for an array of queue pointers starting at q_arr.
1569 */
1570int
1571csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw)
1572{
1573 int i;
1574
1575 if (!wrm->num_q) {
1576 csio_err(hw, "Num queues is not set\n");
1577 return -EINVAL;
1578 }
1579
1580 wrm->q_arr = kzalloc(sizeof(struct csio_q *) * wrm->num_q, GFP_KERNEL);
1581 if (!wrm->q_arr)
1582 goto err;
1583
1584 for (i = 0; i < wrm->num_q; i++) {
1585 wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL);
1586 if (!wrm->q_arr[i]) {
1587 while (--i >= 0)
1588 kfree(wrm->q_arr[i]);
1589 goto err_free_arr;
1590 }
1591 }
1592 wrm->free_qidx = 0;
1593
1594 return 0;
1595
1596err_free_arr:
1597 kfree(wrm->q_arr);
1598err:
1599 return -ENOMEM;
1600}
1601
1602/*
1603 * csio_wrm_exit - Initialize Work request module.
1604 * @wrm: WR module
1605 * @hw: HW module
1606 *
1607 * Uninitialize WR module. Free q_arr and pointers in it.
1608 * We have the additional job of freeing the DMA memory associated
1609 * with the queues.
1610 */
1611void
1612csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw)
1613{
1614 int i;
1615 uint32_t j;
1616 struct csio_q *q;
1617 struct csio_dma_buf *buf;
1618
1619 for (i = 0; i < wrm->num_q; i++) {
1620 q = wrm->q_arr[i];
1621
1622 if (wrm->free_qidx && (i < wrm->free_qidx)) {
1623 if (q->type == CSIO_FREELIST) {
1624 if (!q->un.fl.bufs)
1625 continue;
1626 for (j = 0; j < q->credits; j++) {
1627 buf = &q->un.fl.bufs[j];
1628 if (!buf->vaddr)
1629 continue;
1630 pci_free_consistent(hw->pdev, buf->len,
1631 buf->vaddr,
1632 buf->paddr);
1633 }
1634 kfree(q->un.fl.bufs);
1635 }
1636 pci_free_consistent(hw->pdev, q->size,
1637 q->vstart, q->pstart);
1638 }
1639 kfree(q);
1640 }
1641
1642 hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED;
1643
1644 kfree(wrm->q_arr);
1645}
Ubuntu Artful kernel mirror