]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/ethernet/cavium/liquidio/octeon_droq.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branches 'asoc/topic/atmel', 'asoc/topic/bcm2835' and 'asoc...
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / cavium / liquidio / octeon_droq.c
CommitLineData
f21fb3ed
RV		 1/**********************************************************************
2* Author: Cavium, Inc.
3*
4* Contact: support@cavium.com
5* Please include "LiquidIO" in the subject.
6*
7* Copyright (c) 2003-2015 Cavium, Inc.
8*
9* This file is free software; you can redistribute it and/or modify
10* it under the terms of the GNU General Public License, Version 2, as
11* published by the Free Software Foundation.
12*
13* This file is distributed in the hope that it will be useful, but
14* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16* NONINFRINGEMENT. See the GNU General Public License for more
17* details.
18*
19* This file may also be available under a different license from Cavium.
20* Contact Cavium, Inc. for more information
21**********************************************************************/
22#include <linux/version.h>
23#include <linux/types.h>
24#include <linux/list.h>
25#include <linux/pci.h>
26#include <linux/kthread.h>
27#include <linux/netdevice.h>
5b173cf9 28#include <linux/vmalloc.h>
f21fb3ed
RV		 29#include "octeon_config.h"
30#include "liquidio_common.h"
31#include "octeon_droq.h"
32#include "octeon_iq.h"
33#include "response_manager.h"
34#include "octeon_device.h"
35#include "octeon_nic.h"
36#include "octeon_main.h"
37#include "octeon_network.h"
38#include "cn66xx_regs.h"
39#include "cn66xx_device.h"
40#include "cn68xx_regs.h"
41#include "cn68xx_device.h"
42#include "liquidio_image.h"
43#include "octeon_mem_ops.h"
44
45/* #define CAVIUM_ONLY_PERF_MODE */
46
47#define CVM_MIN(d1, d2) (((d1) < (d2)) ? (d1) : (d2))
48#define CVM_MAX(d1, d2) (((d1) > (d2)) ? (d1) : (d2))
49
50struct niclist {
51 struct list_head list;
52 void *ptr;
53};
54
55struct __dispatch {
56 struct list_head list;
57 struct octeon_recv_info *rinfo;
58 octeon_dispatch_fn_t disp_fn;
59};
60
61/** Get the argument that the user set when registering dispatch
62 * function for a given opcode/subcode.
63 * @param octeon_dev - the octeon device pointer.
64 * @param opcode - the opcode for which the dispatch argument
65 * is to be checked.
66 * @param subcode - the subcode for which the dispatch argument
67 * is to be checked.
68 * @return Success: void * (argument to the dispatch function)
69 * @return Failure: NULL
70 *
71 */
72static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
73 u16 opcode, u16 subcode)
74{
75 int idx;
76 struct list_head *dispatch;
77 void *fn_arg = NULL;
78 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
79
80 idx = combined_opcode & OCTEON_OPCODE_MASK;
81
82 spin_lock_bh(&octeon_dev->dispatch.lock);
83
84 if (octeon_dev->dispatch.count == 0) {
85 spin_unlock_bh(&octeon_dev->dispatch.lock);
86 return NULL;
87 }
88
89 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
90 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
91 } else {
92 list_for_each(dispatch,
93 &octeon_dev->dispatch.dlist[idx].list) {
94 if (((struct octeon_dispatch *)dispatch)->opcode ==
95 combined_opcode) {
96 fn_arg = ((struct octeon_dispatch *)
97 dispatch)->arg;
98 break;
99 }
100 }
101 }
102
103 spin_unlock_bh(&octeon_dev->dispatch.lock);
104 return fn_arg;
105}
106
107u32 octeon_droq_check_hw_for_pkts(struct octeon_device *oct,
108 struct octeon_droq *droq)
109{
110 u32 pkt_count = 0;
111
112 pkt_count = readl(droq->pkts_sent_reg);
113 if (pkt_count) {
114 atomic_add(pkt_count, &droq->pkts_pending);
115 writel(pkt_count, droq->pkts_sent_reg);
116 }
117
118 return pkt_count;
119}
120
121static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
122{
123 u32 count = 0;
124
125 /* max_empty_descs is the max. no. of descs that can have no buffers.
126 * If the empty desc count goes beyond this value, we cannot safely
127 * read in a 64K packet sent by Octeon
128 * (64K is max pkt size from Octeon)
129 */
130 droq->max_empty_descs = 0;
131
132 do {
133 droq->max_empty_descs++;
134 count += droq->buffer_size;
135 } while (count < (64 * 1024));
136
137 droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
138}
139
140static void octeon_droq_reset_indices(struct octeon_droq *droq)
141{
142 droq->read_idx = 0;
143 droq->write_idx = 0;
144 droq->refill_idx = 0;
145 droq->refill_count = 0;
146 atomic_set(&droq->pkts_pending, 0);
147}
148
149static void
150octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
151 struct octeon_droq *droq)
152{
153 u32 i;
154
155 for (i = 0; i < droq->max_count; i++) {
156 if (droq->recv_buf_list[i].buffer) {
157 if (droq->desc_ring) {
158 lio_unmap_ring_info(oct->pci_dev,
159 (u64)droq->
160 desc_ring[i].info_ptr,
161 OCT_DROQ_INFO_SIZE);
162 lio_unmap_ring(oct->pci_dev,
163 (u64)droq->desc_ring[i].
164 buffer_ptr,
165 droq->buffer_size);
166 }
167 recv_buffer_free(droq->recv_buf_list[i].buffer);
168 droq->recv_buf_list[i].buffer = NULL;
169 }
170 }
171
172 octeon_droq_reset_indices(droq);
173}
174
175static int
176octeon_droq_setup_ring_buffers(struct octeon_device *oct,
177 struct octeon_droq *droq)
178{
179 u32 i;
180 void *buf;
181 struct octeon_droq_desc *desc_ring = droq->desc_ring;
182
183 for (i = 0; i < droq->max_count; i++) {
184 buf = recv_buffer_alloc(oct, droq->q_no, droq->buffer_size);
185
186 if (!buf) {
187 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
188 __func__);
189 return -ENOMEM;
190 }
191
192 droq->recv_buf_list[i].buffer = buf;
193 droq->recv_buf_list[i].data = get_rbd(buf);
194
195 droq->info_list[i].length = 0;
196
197 /* map ring buffers into memory */
198 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
199 desc_ring[i].buffer_ptr =
200 lio_map_ring(oct->pci_dev,
201 droq->recv_buf_list[i].buffer,
202 droq->buffer_size);
203 }
204
205 octeon_droq_reset_indices(droq);
206
207 octeon_droq_compute_max_packet_bufs(droq);
208
209 return 0;
210}
211
212int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
213{
214 struct octeon_droq *droq = oct->droq[q_no];
215
216 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
217
218 octeon_droq_destroy_ring_buffers(oct, droq);
9686f310 219 vfree(droq->recv_buf_list);
f21fb3ed
RV		 220
221 if (droq->info_base_addr)
222 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list,
223 droq->info_alloc_size,
224 droq->info_base_addr,
225 droq->info_list_dma);
226
227 if (droq->desc_ring)
228 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
229 droq->desc_ring, droq->desc_ring_dma);
230
231 memset(droq, 0, OCT_DROQ_SIZE);
232
233 return 0;
234}
235
236int octeon_init_droq(struct octeon_device *oct,
237 u32 q_no,
238 u32 num_descs,
239 u32 desc_size,
240 void *app_ctx)
241{
242 struct octeon_droq *droq;
243 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
244 u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
245
246 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
247
248 droq = oct->droq[q_no];
249 memset(droq, 0, OCT_DROQ_SIZE);
250
251 droq->oct_dev = oct;
252 droq->q_no = q_no;
253 if (app_ctx)
254 droq->app_ctx = app_ctx;
255 else
256 droq->app_ctx = (void *)(size_t)q_no;
257
258 c_num_descs = num_descs;
259 c_buf_size = desc_size;
260 if (OCTEON_CN6XXX(oct)) {
261 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf);
262
263 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
264 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
265 }
266
267 droq->max_count = c_num_descs;
268 droq->buffer_size = c_buf_size;
269
270 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
271 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
272 (dma_addr_t *)&droq->desc_ring_dma);
273
274 if (!droq->desc_ring) {
275 dev_err(&oct->pci_dev->dev,
276 "Output queue %d ring alloc failed\n", q_no);
277 return 1;
278 }
279
280 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
281 q_no, droq->desc_ring, droq->desc_ring_dma);
282 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
283 droq->max_count);
284
285 droq->info_list =
286 cnnic_alloc_aligned_dma(oct->pci_dev,
287 (droq->max_count * OCT_DROQ_INFO_SIZE),
288 &droq->info_alloc_size,
289 &droq->info_base_addr,
290 &droq->info_list_dma);
291
292 if (!droq->info_list) {
293 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n");
294 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
295 droq->desc_ring, droq->desc_ring_dma);
296 return 1;
297 }
298
299 droq->recv_buf_list = (struct octeon_recv_buffer *)
300 vmalloc(droq->max_count *
301 OCT_DROQ_RECVBUF_SIZE);
302 if (!droq->recv_buf_list) {
303 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
304 goto init_droq_fail;
305 }
306
307 if (octeon_droq_setup_ring_buffers(oct, droq))
308 goto init_droq_fail;
309
310 droq->pkts_per_intr = c_pkts_per_intr;
311 droq->refill_threshold = c_refill_threshold;
312
313 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
314 droq->max_empty_descs);
315
316 spin_lock_init(&droq->lock);
317
318 INIT_LIST_HEAD(&droq->dispatch_list);
319
320 /* For 56xx Pass1, this function won't be called, so no checks. */
321 oct->fn_list.setup_oq_regs(oct, q_no);
322
323 oct->io_qmask.oq |= (1 << q_no);
324
325 return 0;
326
327init_droq_fail:
328 octeon_delete_droq(oct, q_no);
329 return 1;
330}
331
332/* octeon_create_recv_info
333 * Parameters:
334 * octeon_dev - pointer to the octeon device structure
335 * droq - droq in which the packet arrived.
336 * buf_cnt - no. of buffers used by the packet.
337 * idx - index in the descriptor for the first buffer in the packet.
338 * Description:
339 * Allocates a recv_info_t and copies the buffer addresses for packet data
340 * into the recv_pkt space which starts at an 8B offset from recv_info_t.
341 * Flags the descriptors for refill later. If available descriptors go
342 * below the threshold to receive a 64K pkt, new buffers are first allocated
343 * before the recv_pkt_t is created.
344 * This routine will be called in interrupt context.
345 * Returns:
346 * Success: Pointer to recv_info_t
347 * Failure: NULL.
348 * Locks:
349 * The droq->lock is held when this routine is called.
350 */
351static inline struct octeon_recv_info *octeon_create_recv_info(
352 struct octeon_device *octeon_dev,
353 struct octeon_droq *droq,
354 u32 buf_cnt,
355 u32 idx)
356{
357 struct octeon_droq_info *info;
358 struct octeon_recv_pkt *recv_pkt;
359 struct octeon_recv_info *recv_info;
360 u32 i, bytes_left;
361
362 info = &droq->info_list[idx];
363
364 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
365 if (!recv_info)
366 return NULL;
367
368 recv_pkt = recv_info->recv_pkt;
369 recv_pkt->rh = info->rh;
370 recv_pkt->length = (u32)info->length;
371 recv_pkt->buffer_count = (u16)buf_cnt;
372 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
373
374 i = 0;
375 bytes_left = (u32)info->length;
376
377 while (buf_cnt) {
378 lio_unmap_ring(octeon_dev->pci_dev,
379 (u64)droq->desc_ring[idx].buffer_ptr,
380 droq->buffer_size);
381
382 recv_pkt->buffer_size[i] =
383 (bytes_left >=
384 droq->buffer_size) ? droq->buffer_size : bytes_left;
385
386 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
387 droq->recv_buf_list[idx].buffer = NULL;
388
389 INCR_INDEX_BY1(idx, droq->max_count);
390 bytes_left -= droq->buffer_size;
391 i++;
392 buf_cnt--;
393 }
394
395 return recv_info;
396}
397
398/* If we were not able to refill all buffers, try to move around
399 * the buffers that were not dispatched.
400 */
401static inline u32
402octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
403 struct octeon_droq_desc *desc_ring)
404{
405 u32 desc_refilled = 0;
406
407 u32 refill_index = droq->refill_idx;
408
409 while (refill_index != droq->read_idx) {
410 if (droq->recv_buf_list[refill_index].buffer) {
411 droq->recv_buf_list[droq->refill_idx].buffer =
412 droq->recv_buf_list[refill_index].buffer;
413 droq->recv_buf_list[droq->refill_idx].data =
414 droq->recv_buf_list[refill_index].data;
415 desc_ring[droq->refill_idx].buffer_ptr =
416 desc_ring[refill_index].buffer_ptr;
417 droq->recv_buf_list[refill_index].buffer = NULL;
418 desc_ring[refill_index].buffer_ptr = 0;
419 do {
420 INCR_INDEX_BY1(droq->refill_idx,
421 droq->max_count);
422 desc_refilled++;
423 droq->refill_count--;
424 } while (droq->recv_buf_list[droq->refill_idx].
425 buffer);
426 }
427 INCR_INDEX_BY1(refill_index, droq->max_count);
428 } /* while */
429 return desc_refilled;
430}
431
432/* octeon_droq_refill
433 * Parameters:
434 * droq - droq in which descriptors require new buffers.
435 * Description:
436 * Called during normal DROQ processing in interrupt mode or by the poll
437 * thread to refill the descriptors from which buffers were dispatched
438 * to upper layers. Attempts to allocate new buffers. If that fails, moves
439 * up buffers (that were not dispatched) to form a contiguous ring.
440 * Returns:
441 * No of descriptors refilled.
442 * Locks:
443 * This routine is called with droq->lock held.
444 */
445static u32
446octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
447{
448 struct octeon_droq_desc *desc_ring;
449 void *buf = NULL;
450 u8 *data;
451 u32 desc_refilled = 0;
452
453 desc_ring = droq->desc_ring;
454
455 while (droq->refill_count && (desc_refilled < droq->max_count)) {
456 /* If a valid buffer exists (happens if there is no dispatch),
457 * reuse
458 * the buffer, else allocate.
459 */
460 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
461 buf = recv_buffer_alloc(octeon_dev, droq->q_no,
462 droq->buffer_size);
463 /* If a buffer could not be allocated, no point in
464 * continuing
465 */
466 if (!buf)
467 break;
468 droq->recv_buf_list[droq->refill_idx].buffer =
469 buf;
470 data = get_rbd(buf);
471 } else {
472 data = get_rbd(droq->recv_buf_list
473 [droq->refill_idx].buffer);
474 }
475
476 droq->recv_buf_list[droq->refill_idx].data = data;
477
478 desc_ring[droq->refill_idx].buffer_ptr =
479 lio_map_ring(octeon_dev->pci_dev,
480 droq->recv_buf_list[droq->
481 refill_idx].buffer,
482 droq->buffer_size);
483
484 /* Reset any previous values in the length field. */
485 droq->info_list[droq->refill_idx].length = 0;
486
487 INCR_INDEX_BY1(droq->refill_idx, droq->max_count);
488 desc_refilled++;
489 droq->refill_count--;
490 }
491
492 if (droq->refill_count)
493 desc_refilled +=
494 octeon_droq_refill_pullup_descs(droq, desc_ring);
495
496 /* if droq->refill_count
497 * The refill count would not change in pass two. We only moved buffers
498 * to close the gap in the ring, but we would still have the same no. of
499 * buffers to refill.
500 */
501 return desc_refilled;
502}
503
504static inline u32
505octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
506{
507 u32 buf_cnt = 0;
508
509 while (total_len > (buf_size * buf_cnt))
510 buf_cnt++;
511 return buf_cnt;
512}
513
514static int
515octeon_droq_dispatch_pkt(struct octeon_device *oct,
516 struct octeon_droq *droq,
517 union octeon_rh *rh,
518 struct octeon_droq_info *info)
519{
520 u32 cnt;
521 octeon_dispatch_fn_t disp_fn;
522 struct octeon_recv_info *rinfo;
523
524 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
525
526 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
527 (u16)rh->r.subcode);
528 if (disp_fn) {
529 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
530 if (rinfo) {
531 struct __dispatch *rdisp = rinfo->rsvd;
532
533 rdisp->rinfo = rinfo;
534 rdisp->disp_fn = disp_fn;
535 rinfo->recv_pkt->rh = *rh;
536 list_add_tail(&rdisp->list,
537 &droq->dispatch_list);
538 } else {
539 droq->stats.dropped_nomem++;
540 }
541 } else {
542 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function\n");
543 droq->stats.dropped_nodispatch++;
544 } /* else (dispatch_fn ... */
545
546 return cnt;
547}
548
549static inline void octeon_droq_drop_packets(struct octeon_device *oct,
550 struct octeon_droq *droq,
551 u32 cnt)
552{
553 u32 i = 0, buf_cnt;
554 struct octeon_droq_info *info;
555
556 for (i = 0; i < cnt; i++) {
557 info = &droq->info_list[droq->read_idx];
558 octeon_swap_8B_data((u64 *)info, 2);
559
560 if (info->length) {
561 info->length -= OCT_RH_SIZE;
562 droq->stats.bytes_received += info->length;
563 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
564 (u32)info->length);
565 } else {
566 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
567 buf_cnt = 1;
568 }
569
570 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
571 droq->refill_count += buf_cnt;
572 }
573}
574
575static u32
576octeon_droq_fast_process_packets(struct octeon_device *oct,
577 struct octeon_droq *droq,
578 u32 pkts_to_process)
579{
580 struct octeon_droq_info *info;
581 union octeon_rh *rh;
582 u32 pkt, total_len = 0, pkt_count;
583
584 pkt_count = pkts_to_process;
585
586 for (pkt = 0; pkt < pkt_count; pkt++) {
587 u32 pkt_len = 0;
588 struct sk_buff *nicbuf = NULL;
589
590 info = &droq->info_list[droq->read_idx];
591 octeon_swap_8B_data((u64 *)info, 2);
592
593 if (!info->length) {
594 dev_err(&oct->pci_dev->dev,
595 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
596 droq->q_no, droq->read_idx, pkt_count);
597 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
598 (u8 *)info,
599 OCT_DROQ_INFO_SIZE);
600 break;
601 }
602
603 /* Len of resp hdr in included in the received data len. */
604 info->length -= OCT_RH_SIZE;
605 rh = &info->rh;
606
607 total_len += (u32)info->length;
608
609 if (OPCODE_SLOW_PATH(rh)) {
610 u32 buf_cnt;
611
612 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
613 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
614 droq->refill_count += buf_cnt;
615 } else {
616 if (info->length <= droq->buffer_size) {
617 lio_unmap_ring(oct->pci_dev,
618 (u64)droq->desc_ring[
619 droq->read_idx].buffer_ptr,
620 droq->buffer_size);
621 pkt_len = (u32)info->length;
622 nicbuf = droq->recv_buf_list[
623 droq->read_idx].buffer;
624 droq->recv_buf_list[droq->read_idx].buffer =
625 NULL;
626 INCR_INDEX_BY1(droq->read_idx, droq->max_count);
627 skb_put(nicbuf, pkt_len);
628 droq->refill_count++;
629 } else {
630 nicbuf = octeon_fast_packet_alloc(oct, droq,
631 droq->q_no,
632 (u32)
633 info->length);
634 pkt_len = 0;
635 /* nicbuf allocation can fail. We'll handle it
636 * inside the loop.
637 */
638 while (pkt_len < info->length) {
639 int cpy_len;
640
641 cpy_len = ((pkt_len +
642 droq->buffer_size) >
643 info->length) ?
644 ((u32)info->length - pkt_len) :
645 droq->buffer_size;
646
647 if (nicbuf) {
648 lio_unmap_ring(oct->pci_dev,
649 (u64)
650 droq->desc_ring
651 [droq->read_idx].
652 buffer_ptr,
653 droq->
654 buffer_size);
655 octeon_fast_packet_next(droq,
656 nicbuf,
657 cpy_len,
658 droq->
659 read_idx
660 );
661 }
662
663 pkt_len += cpy_len;
664 INCR_INDEX_BY1(droq->read_idx,
665 droq->max_count);
666 droq->refill_count++;
667 }
668 }
669
670 if (nicbuf) {
671 if (droq->ops.fptr)
672 droq->ops.fptr(oct->octeon_id,
673 nicbuf, pkt_len,
674 rh, &droq->napi);
675 else
676 recv_buffer_free(nicbuf);
677 }
678 }
679
680 if (droq->refill_count >= droq->refill_threshold) {
681 int desc_refilled = octeon_droq_refill(oct, droq);
682
683 /* Flush the droq descriptor data to memory to be sure
684 * that when we update the credits the data in memory
685 * is accurate.
686 */
687 wmb();
688 writel((desc_refilled), droq->pkts_credit_reg);
689 /* make sure mmio write completes */
690 mmiowb();
691 }
692
693 } /* for ( each packet )... */
694
695 /* Increment refill_count by the number of buffers processed. */
696 droq->stats.pkts_received += pkt;
697 droq->stats.bytes_received += total_len;
698
699 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
700 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
701
702 droq->stats.dropped_toomany += (pkts_to_process - pkt);
703 return pkts_to_process;
704 }
705
706 return pkt;
707}
708
709int
710octeon_droq_process_packets(struct octeon_device *oct,
711 struct octeon_droq *droq,
712 u32 budget)
713{
714 u32 pkt_count = 0, pkts_processed = 0;
715 struct list_head *tmp, *tmp2;
716
717 pkt_count = atomic_read(&droq->pkts_pending);
718 if (!pkt_count)
719 return 0;
720
721 if (pkt_count > budget)
722 pkt_count = budget;
723
724 /* Grab the lock */
725 spin_lock(&droq->lock);
726
727 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
728
729 atomic_sub(pkts_processed, &droq->pkts_pending);
730
731 /* Release the spin lock */
732 spin_unlock(&droq->lock);
733
734 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
735 struct __dispatch *rdisp = (struct __dispatch *)tmp;
736
737 list_del(tmp);
738 rdisp->disp_fn(rdisp->rinfo,
739 octeon_get_dispatch_arg
740 (oct,
741 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
742 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
743 }
744
745 /* If there are packets pending. schedule tasklet again */
746 if (atomic_read(&droq->pkts_pending))
747 return 1;
748
749 return 0;
750}
751
752/**
753 * Utility function to poll for packets. check_hw_for_packets must be
754 * called before calling this routine.
755 */
756
757static int
758octeon_droq_process_poll_pkts(struct octeon_device *oct,
759 struct octeon_droq *droq, u32 budget)
760{
761 struct list_head *tmp, *tmp2;
762 u32 pkts_available = 0, pkts_processed = 0;
763 u32 total_pkts_processed = 0;
764
765 if (budget > droq->max_count)
766 budget = droq->max_count;
767
768 spin_lock(&droq->lock);
769
770 while (total_pkts_processed < budget) {
771 pkts_available =
772 CVM_MIN((budget - total_pkts_processed),
773 (u32)(atomic_read(&droq->pkts_pending)));
774
775 if (pkts_available == 0)
776 break;
777
778 pkts_processed =
779 octeon_droq_fast_process_packets(oct, droq,
780 pkts_available);
781
782 atomic_sub(pkts_processed, &droq->pkts_pending);
783
784 total_pkts_processed += pkts_processed;
785
786 octeon_droq_check_hw_for_pkts(oct, droq);
787 }
788
789 spin_unlock(&droq->lock);
790
791 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
792 struct __dispatch *rdisp = (struct __dispatch *)tmp;
793
794 list_del(tmp);
795 rdisp->disp_fn(rdisp->rinfo,
796 octeon_get_dispatch_arg
797 (oct,
798 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
799 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
800 }
801
802 return total_pkts_processed;
803}
804
805int
806octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd,
807 u32 arg)
808{
809 struct octeon_droq *droq;
810 struct octeon_config *oct_cfg = NULL;
811
812 oct_cfg = octeon_get_conf(oct);
813
814 if (!oct_cfg)
815 return -EINVAL;
816
817 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
818 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
819 __func__, q_no, (oct->num_oqs - 1));
820 return -EINVAL;
821 }
822
823 droq = oct->droq[q_no];
824
825 if (cmd == POLL_EVENT_PROCESS_PKTS)
826 return octeon_droq_process_poll_pkts(oct, droq, arg);
827
828 if (cmd == POLL_EVENT_PENDING_PKTS) {
829 u32 pkt_cnt = atomic_read(&droq->pkts_pending);
830
831 return octeon_droq_process_packets(oct, droq, pkt_cnt);
832 }
833
834 if (cmd == POLL_EVENT_ENABLE_INTR) {
835 u32 value;
836 unsigned long flags;
837
838 /* Enable Pkt Interrupt */
839 switch (oct->chip_id) {
840 case OCTEON_CN66XX:
841 case OCTEON_CN68XX: {
842 struct octeon_cn6xxx *cn6xxx =
843 (struct octeon_cn6xxx *)oct->chip;
844 spin_lock_irqsave
845 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
846 value =
847 octeon_read_csr(oct,
848 CN6XXX_SLI_PKT_TIME_INT_ENB);
849 value |= (1 << q_no);
850 octeon_write_csr(oct,
851 CN6XXX_SLI_PKT_TIME_INT_ENB,
852 value);
853 value =
854 octeon_read_csr(oct,
855 CN6XXX_SLI_PKT_CNT_INT_ENB);
856 value |= (1 << q_no);
857 octeon_write_csr(oct,
858 CN6XXX_SLI_PKT_CNT_INT_ENB,
859 value);
860
861 /* don't bother flushing the enables */
862
863 spin_unlock_irqrestore
864 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
865 return 0;
866 }
867 break;
868 }
869
870 return 0;
871 }
872
873 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd);
874 return -EINVAL;
875}
876
877int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
878 struct octeon_droq_ops *ops)
879{
880 struct octeon_droq *droq;
881 unsigned long flags;
882 struct octeon_config *oct_cfg = NULL;
883
884 oct_cfg = octeon_get_conf(oct);
885
886 if (!oct_cfg)
887 return -EINVAL;
888
889 if (!(ops)) {
890 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
891 __func__);
892 return -EINVAL;
893 }
894
895 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
896 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
897 __func__, q_no, (oct->num_oqs - 1));
898 return -EINVAL;
899 }
900
901 droq = oct->droq[q_no];
902
903 spin_lock_irqsave(&droq->lock, flags);
904
905 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
906
907 spin_unlock_irqrestore(&droq->lock, flags);
908
909 return 0;
910}
911
912int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
913{
914 unsigned long flags;
915 struct octeon_droq *droq;
916 struct octeon_config *oct_cfg = NULL;
917
918 oct_cfg = octeon_get_conf(oct);
919
920 if (!oct_cfg)
921 return -EINVAL;
922
923 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
924 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
925 __func__, q_no, oct->num_oqs - 1);
926 return -EINVAL;
927 }
928
929 droq = oct->droq[q_no];
930
931 if (!droq) {
932 dev_info(&oct->pci_dev->dev,
933 "Droq id (%d) not available.\n", q_no);
934 return 0;
935 }
936
937 spin_lock_irqsave(&droq->lock, flags);
938
939 droq->ops.fptr = NULL;
940 droq->ops.drop_on_max = 0;
941
942 spin_unlock_irqrestore(&droq->lock, flags);
943
944 return 0;
945}
946
947int octeon_create_droq(struct octeon_device *oct,
948 u32 q_no, u32 num_descs,
949 u32 desc_size, void *app_ctx)
950{
951 struct octeon_droq *droq;
952
953 if (oct->droq[q_no]) {
954 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
955 q_no);
956 return 1;
957 }
958
959 /* Allocate the DS for the new droq. */
960 droq = vmalloc(sizeof(*droq));
961 if (!droq)
962 goto create_droq_fail;
963 memset(droq, 0, sizeof(struct octeon_droq));
964
965 /*Disable the pkt o/p for this Q */
966 octeon_set_droq_pkt_op(oct, q_no, 0);
967 oct->droq[q_no] = droq;
968
969 /* Initialize the Droq */
970 octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx);
971
972 oct->num_oqs++;
973
974 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
975 oct->num_oqs);
976
977 /* Global Droq register settings */
978
979 /* As of now not required, as setting are done for all 32 Droqs at
980 * the same time.
981 */
982 return 0;
983
984create_droq_fail:
985 octeon_delete_droq(oct, q_no);
08a965ec 986 return -ENOMEM;
f21fb3ed 987}
ubuntu-zesty-kernel mirror