]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/net/ethernet/cavium/liquidio/octeon_droq.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
spi-imx: Implements handling of the SPI_READY mode flag.
[mirror_ubuntu-bionic-kernel.git] / drivers / net / ethernet / cavium / liquidio / octeon_droq.c
1 /**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2016 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more details.
17 ***********************************************************************/
18 #include <linux/pci.h>
19 #include <linux/netdevice.h>
20 #include <linux/vmalloc.h>
21 #include "liquidio_common.h"
22 #include "octeon_droq.h"
23 #include "octeon_iq.h"
24 #include "response_manager.h"
25 #include "octeon_device.h"
26 #include "octeon_main.h"
27 #include "octeon_network.h"
28 #include "cn66xx_regs.h"
29 #include "cn66xx_device.h"
30 #include "cn23xx_pf_device.h"
31 #include "cn23xx_vf_device.h"
32
33 struct niclist {
34 struct list_head list;
35 void *ptr;
36 };
37
38 struct __dispatch {
39 struct list_head list;
40 struct octeon_recv_info *rinfo;
41 octeon_dispatch_fn_t disp_fn;
42 };
43
44 /** Get the argument that the user set when registering dispatch
45 * function for a given opcode/subcode.
46 * @param octeon_dev - the octeon device pointer.
47 * @param opcode - the opcode for which the dispatch argument
48 * is to be checked.
49 * @param subcode - the subcode for which the dispatch argument
50 * is to be checked.
51 * @return Success: void * (argument to the dispatch function)
52 * @return Failure: NULL
53 *
54 */
55 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
56 u16 opcode, u16 subcode)
57 {
58 int idx;
59 struct list_head *dispatch;
60 void *fn_arg = NULL;
61 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
62
63 idx = combined_opcode & OCTEON_OPCODE_MASK;
64
65 spin_lock_bh(&octeon_dev->dispatch.lock);
66
67 if (octeon_dev->dispatch.count == 0) {
68 spin_unlock_bh(&octeon_dev->dispatch.lock);
69 return NULL;
70 }
71
72 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
73 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
74 } else {
75 list_for_each(dispatch,
76 &octeon_dev->dispatch.dlist[idx].list) {
77 if (((struct octeon_dispatch *)dispatch)->opcode ==
78 combined_opcode) {
79 fn_arg = ((struct octeon_dispatch *)
80 dispatch)->arg;
81 break;
82 }
83 }
84 }
85
86 spin_unlock_bh(&octeon_dev->dispatch.lock);
87 return fn_arg;
88 }
89
90 /** Check for packets on Droq. This function should be called with lock held.
91 * @param droq - Droq on which count is checked.
92 * @return Returns packet count.
93 */
94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
95 {
96 u32 pkt_count = 0;
97 u32 last_count;
98
99 pkt_count = readl(droq->pkts_sent_reg);
100
101 last_count = pkt_count - droq->pkt_count;
102 droq->pkt_count = pkt_count;
103
104 /* we shall write to cnts at napi irq enable or end of droq tasklet */
105 if (last_count)
106 atomic_add(last_count, &droq->pkts_pending);
107
108 return last_count;
109 }
110
111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
112 {
113 u32 count = 0;
114
115 /* max_empty_descs is the max. no. of descs that can have no buffers.
116 * If the empty desc count goes beyond this value, we cannot safely
117 * read in a 64K packet sent by Octeon
118 * (64K is max pkt size from Octeon)
119 */
120 droq->max_empty_descs = 0;
121
122 do {
123 droq->max_empty_descs++;
124 count += droq->buffer_size;
125 } while (count < (64 * 1024));
126
127 droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
128 }
129
130 static void octeon_droq_reset_indices(struct octeon_droq *droq)
131 {
132 droq->read_idx = 0;
133 droq->write_idx = 0;
134 droq->refill_idx = 0;
135 droq->refill_count = 0;
136 atomic_set(&droq->pkts_pending, 0);
137 }
138
139 static void
140 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
141 struct octeon_droq *droq)
142 {
143 u32 i;
144 struct octeon_skb_page_info *pg_info;
145
146 for (i = 0; i < droq->max_count; i++) {
147 pg_info = &droq->recv_buf_list[i].pg_info;
148
149 if (pg_info->dma)
150 lio_unmap_ring(oct->pci_dev,
151 (u64)pg_info->dma);
152 pg_info->dma = 0;
153
154 if (pg_info->page)
155 recv_buffer_destroy(droq->recv_buf_list[i].buffer,
156 pg_info);
157
158 if (droq->desc_ring && droq->desc_ring[i].info_ptr)
159 lio_unmap_ring_info(oct->pci_dev,
160 (u64)droq->
161 desc_ring[i].info_ptr,
162 OCT_DROQ_INFO_SIZE);
163 droq->recv_buf_list[i].buffer = NULL;
164 }
165
166 octeon_droq_reset_indices(droq);
167 }
168
169 static int
170 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
171 struct octeon_droq *droq)
172 {
173 u32 i;
174 void *buf;
175 struct octeon_droq_desc *desc_ring = droq->desc_ring;
176
177 for (i = 0; i < droq->max_count; i++) {
178 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
179
180 if (!buf) {
181 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
182 __func__);
183 droq->stats.rx_alloc_failure++;
184 return -ENOMEM;
185 }
186
187 droq->recv_buf_list[i].buffer = buf;
188 droq->recv_buf_list[i].data = get_rbd(buf);
189 droq->info_list[i].length = 0;
190
191 /* map ring buffers into memory */
192 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
193 desc_ring[i].buffer_ptr =
194 lio_map_ring(droq->recv_buf_list[i].buffer);
195 }
196
197 octeon_droq_reset_indices(droq);
198
199 octeon_droq_compute_max_packet_bufs(droq);
200
201 return 0;
202 }
203
204 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
205 {
206 struct octeon_droq *droq = oct->droq[q_no];
207
208 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
209
210 octeon_droq_destroy_ring_buffers(oct, droq);
211 vfree(droq->recv_buf_list);
212
213 if (droq->info_base_addr)
214 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list,
215 droq->info_alloc_size,
216 droq->info_base_addr,
217 droq->info_list_dma);
218
219 if (droq->desc_ring)
220 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
221 droq->desc_ring, droq->desc_ring_dma);
222
223 memset(droq, 0, OCT_DROQ_SIZE);
224
225 return 0;
226 }
227
228 int octeon_init_droq(struct octeon_device *oct,
229 u32 q_no,
230 u32 num_descs,
231 u32 desc_size,
232 void *app_ctx)
233 {
234 struct octeon_droq *droq;
235 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
236 u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
237 int orig_node = dev_to_node(&oct->pci_dev->dev);
238 int numa_node = cpu_to_node(q_no % num_online_cpus());
239
240 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
241
242 droq = oct->droq[q_no];
243 memset(droq, 0, OCT_DROQ_SIZE);
244
245 droq->oct_dev = oct;
246 droq->q_no = q_no;
247 if (app_ctx)
248 droq->app_ctx = app_ctx;
249 else
250 droq->app_ctx = (void *)(size_t)q_no;
251
252 c_num_descs = num_descs;
253 c_buf_size = desc_size;
254 if (OCTEON_CN6XXX(oct)) {
255 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);
256
257 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
258 c_refill_threshold =
259 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
260 } else if (OCTEON_CN23XX_PF(oct)) {
261 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf);
262
263 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
264 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
265 } else if (OCTEON_CN23XX_VF(oct)) {
266 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf);
267
268 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
269 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
270 } else {
271 return 1;
272 }
273
274 droq->max_count = c_num_descs;
275 droq->buffer_size = c_buf_size;
276
277 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
278 set_dev_node(&oct->pci_dev->dev, numa_node);
279 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
280 (dma_addr_t *)&droq->desc_ring_dma);
281 set_dev_node(&oct->pci_dev->dev, orig_node);
282 if (!droq->desc_ring)
283 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
284 (dma_addr_t *)&droq->desc_ring_dma);
285
286 if (!droq->desc_ring) {
287 dev_err(&oct->pci_dev->dev,
288 "Output queue %d ring alloc failed\n", q_no);
289 return 1;
290 }
291
292 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
293 q_no, droq->desc_ring, droq->desc_ring_dma);
294 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
295 droq->max_count);
296
297 droq->info_list =
298 cnnic_numa_alloc_aligned_dma((droq->max_count *
299 OCT_DROQ_INFO_SIZE),
300 &droq->info_alloc_size,
301 &droq->info_base_addr,
302 numa_node);
303 if (!droq->info_list) {
304 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n");
305 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
306 droq->desc_ring, droq->desc_ring_dma);
307 return 1;
308 }
309
310 droq->recv_buf_list = (struct octeon_recv_buffer *)
311 vmalloc_node(droq->max_count *
312 OCT_DROQ_RECVBUF_SIZE,
313 numa_node);
314 if (!droq->recv_buf_list)
315 droq->recv_buf_list = (struct octeon_recv_buffer *)
316 vmalloc(droq->max_count *
317 OCT_DROQ_RECVBUF_SIZE);
318 if (!droq->recv_buf_list) {
319 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
320 goto init_droq_fail;
321 }
322
323 if (octeon_droq_setup_ring_buffers(oct, droq))
324 goto init_droq_fail;
325
326 droq->pkts_per_intr = c_pkts_per_intr;
327 droq->refill_threshold = c_refill_threshold;
328
329 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
330 droq->max_empty_descs);
331
332 spin_lock_init(&droq->lock);
333
334 INIT_LIST_HEAD(&droq->dispatch_list);
335
336 /* For 56xx Pass1, this function won't be called, so no checks. */
337 oct->fn_list.setup_oq_regs(oct, q_no);
338
339 oct->io_qmask.oq |= BIT_ULL(q_no);
340
341 return 0;
342
343 init_droq_fail:
344 octeon_delete_droq(oct, q_no);
345 return 1;
346 }
347
348 /* octeon_create_recv_info
349 * Parameters:
350 * octeon_dev - pointer to the octeon device structure
351 * droq - droq in which the packet arrived.
352 * buf_cnt - no. of buffers used by the packet.
353 * idx - index in the descriptor for the first buffer in the packet.
354 * Description:
355 * Allocates a recv_info_t and copies the buffer addresses for packet data
356 * into the recv_pkt space which starts at an 8B offset from recv_info_t.
357 * Flags the descriptors for refill later. If available descriptors go
358 * below the threshold to receive a 64K pkt, new buffers are first allocated
359 * before the recv_pkt_t is created.
360 * This routine will be called in interrupt context.
361 * Returns:
362 * Success: Pointer to recv_info_t
363 * Failure: NULL.
364 * Locks:
365 * The droq->lock is held when this routine is called.
366 */
367 static inline struct octeon_recv_info *octeon_create_recv_info(
368 struct octeon_device *octeon_dev,
369 struct octeon_droq *droq,
370 u32 buf_cnt,
371 u32 idx)
372 {
373 struct octeon_droq_info *info;
374 struct octeon_recv_pkt *recv_pkt;
375 struct octeon_recv_info *recv_info;
376 u32 i, bytes_left;
377 struct octeon_skb_page_info *pg_info;
378
379 info = &droq->info_list[idx];
380
381 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
382 if (!recv_info)
383 return NULL;
384
385 recv_pkt = recv_info->recv_pkt;
386 recv_pkt->rh = info->rh;
387 recv_pkt->length = (u32)info->length;
388 recv_pkt->buffer_count = (u16)buf_cnt;
389 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
390
391 i = 0;
392 bytes_left = (u32)info->length;
393
394 while (buf_cnt) {
395 {
396 pg_info = &droq->recv_buf_list[idx].pg_info;
397
398 lio_unmap_ring(octeon_dev->pci_dev,
399 (u64)pg_info->dma);
400 pg_info->page = NULL;
401 pg_info->dma = 0;
402 }
403
404 recv_pkt->buffer_size[i] =
405 (bytes_left >=
406 droq->buffer_size) ? droq->buffer_size : bytes_left;
407
408 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
409 droq->recv_buf_list[idx].buffer = NULL;
410
411 idx = incr_index(idx, 1, droq->max_count);
412 bytes_left -= droq->buffer_size;
413 i++;
414 buf_cnt--;
415 }
416
417 return recv_info;
418 }
419
420 /* If we were not able to refill all buffers, try to move around
421 * the buffers that were not dispatched.
422 */
423 static inline u32
424 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
425 struct octeon_droq_desc *desc_ring)
426 {
427 u32 desc_refilled = 0;
428
429 u32 refill_index = droq->refill_idx;
430
431 while (refill_index != droq->read_idx) {
432 if (droq->recv_buf_list[refill_index].buffer) {
433 droq->recv_buf_list[droq->refill_idx].buffer =
434 droq->recv_buf_list[refill_index].buffer;
435 droq->recv_buf_list[droq->refill_idx].data =
436 droq->recv_buf_list[refill_index].data;
437 desc_ring[droq->refill_idx].buffer_ptr =
438 desc_ring[refill_index].buffer_ptr;
439 droq->recv_buf_list[refill_index].buffer = NULL;
440 desc_ring[refill_index].buffer_ptr = 0;
441 do {
442 droq->refill_idx = incr_index(droq->refill_idx,
443 1,
444 droq->max_count);
445 desc_refilled++;
446 droq->refill_count--;
447 } while (droq->recv_buf_list[droq->refill_idx].
448 buffer);
449 }
450 refill_index = incr_index(refill_index, 1, droq->max_count);
451 } /* while */
452 return desc_refilled;
453 }
454
455 /* octeon_droq_refill
456 * Parameters:
457 * droq - droq in which descriptors require new buffers.
458 * Description:
459 * Called during normal DROQ processing in interrupt mode or by the poll
460 * thread to refill the descriptors from which buffers were dispatched
461 * to upper layers. Attempts to allocate new buffers. If that fails, moves
462 * up buffers (that were not dispatched) to form a contiguous ring.
463 * Returns:
464 * No of descriptors refilled.
465 * Locks:
466 * This routine is called with droq->lock held.
467 */
468 static u32
469 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
470 {
471 struct octeon_droq_desc *desc_ring;
472 void *buf = NULL;
473 u8 *data;
474 u32 desc_refilled = 0;
475 struct octeon_skb_page_info *pg_info;
476
477 desc_ring = droq->desc_ring;
478
479 while (droq->refill_count && (desc_refilled < droq->max_count)) {
480 /* If a valid buffer exists (happens if there is no dispatch),
481 * reuse
482 * the buffer, else allocate.
483 */
484 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
485 pg_info =
486 &droq->recv_buf_list[droq->refill_idx].pg_info;
487 /* Either recycle the existing pages or go for
488 * new page alloc
489 */
490 if (pg_info->page)
491 buf = recv_buffer_reuse(octeon_dev, pg_info);
492 else
493 buf = recv_buffer_alloc(octeon_dev, pg_info);
494 /* If a buffer could not be allocated, no point in
495 * continuing
496 */
497 if (!buf) {
498 droq->stats.rx_alloc_failure++;
499 break;
500 }
501 droq->recv_buf_list[droq->refill_idx].buffer =
502 buf;
503 data = get_rbd(buf);
504 } else {
505 data = get_rbd(droq->recv_buf_list
506 [droq->refill_idx].buffer);
507 }
508
509 droq->recv_buf_list[droq->refill_idx].data = data;
510
511 desc_ring[droq->refill_idx].buffer_ptr =
512 lio_map_ring(droq->recv_buf_list[droq->
513 refill_idx].buffer);
514 /* Reset any previous values in the length field. */
515 droq->info_list[droq->refill_idx].length = 0;
516
517 droq->refill_idx = incr_index(droq->refill_idx, 1,
518 droq->max_count);
519 desc_refilled++;
520 droq->refill_count--;
521 }
522
523 if (droq->refill_count)
524 desc_refilled +=
525 octeon_droq_refill_pullup_descs(droq, desc_ring);
526
527 /* if droq->refill_count
528 * The refill count would not change in pass two. We only moved buffers
529 * to close the gap in the ring, but we would still have the same no. of
530 * buffers to refill.
531 */
532 return desc_refilled;
533 }
534
535 static inline u32
536 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
537 {
538 u32 buf_cnt = 0;
539
540 while (total_len > (buf_size * buf_cnt))
541 buf_cnt++;
542 return buf_cnt;
543 }
544
545 static int
546 octeon_droq_dispatch_pkt(struct octeon_device *oct,
547 struct octeon_droq *droq,
548 union octeon_rh *rh,
549 struct octeon_droq_info *info)
550 {
551 u32 cnt;
552 octeon_dispatch_fn_t disp_fn;
553 struct octeon_recv_info *rinfo;
554
555 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
556
557 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
558 (u16)rh->r.subcode);
559 if (disp_fn) {
560 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
561 if (rinfo) {
562 struct __dispatch *rdisp = rinfo->rsvd;
563
564 rdisp->rinfo = rinfo;
565 rdisp->disp_fn = disp_fn;
566 rinfo->recv_pkt->rh = *rh;
567 list_add_tail(&rdisp->list,
568 &droq->dispatch_list);
569 } else {
570 droq->stats.dropped_nomem++;
571 }
572 } else {
573 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
574 (unsigned int)rh->r.opcode,
575 (unsigned int)rh->r.subcode);
576 droq->stats.dropped_nodispatch++;
577 }
578
579 return cnt;
580 }
581
582 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
583 struct octeon_droq *droq,
584 u32 cnt)
585 {
586 u32 i = 0, buf_cnt;
587 struct octeon_droq_info *info;
588
589 for (i = 0; i < cnt; i++) {
590 info = &droq->info_list[droq->read_idx];
591 octeon_swap_8B_data((u64 *)info, 2);
592
593 if (info->length) {
594 info->length -= OCT_RH_SIZE;
595 droq->stats.bytes_received += info->length;
596 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
597 (u32)info->length);
598 } else {
599 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
600 buf_cnt = 1;
601 }
602
603 droq->read_idx = incr_index(droq->read_idx, buf_cnt,
604 droq->max_count);
605 droq->refill_count += buf_cnt;
606 }
607 }
608
609 static u32
610 octeon_droq_fast_process_packets(struct octeon_device *oct,
611 struct octeon_droq *droq,
612 u32 pkts_to_process)
613 {
614 struct octeon_droq_info *info;
615 union octeon_rh *rh;
616 u32 pkt, total_len = 0, pkt_count;
617
618 pkt_count = pkts_to_process;
619
620 for (pkt = 0; pkt < pkt_count; pkt++) {
621 u32 pkt_len = 0;
622 struct sk_buff *nicbuf = NULL;
623 struct octeon_skb_page_info *pg_info;
624 void *buf;
625
626 info = &droq->info_list[droq->read_idx];
627 octeon_swap_8B_data((u64 *)info, 2);
628
629 if (!info->length) {
630 dev_err(&oct->pci_dev->dev,
631 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
632 droq->q_no, droq->read_idx, pkt_count);
633 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
634 (u8 *)info,
635 OCT_DROQ_INFO_SIZE);
636 break;
637 }
638
639 /* Len of resp hdr in included in the received data len. */
640 info->length -= OCT_RH_SIZE;
641 rh = &info->rh;
642
643 total_len += (u32)info->length;
644 if (opcode_slow_path(rh)) {
645 u32 buf_cnt;
646
647 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
648 droq->read_idx = incr_index(droq->read_idx,
649 buf_cnt, droq->max_count);
650 droq->refill_count += buf_cnt;
651 } else {
652 if (info->length <= droq->buffer_size) {
653 pkt_len = (u32)info->length;
654 nicbuf = droq->recv_buf_list[
655 droq->read_idx].buffer;
656 pg_info = &droq->recv_buf_list[
657 droq->read_idx].pg_info;
658 if (recv_buffer_recycle(oct, pg_info))
659 pg_info->page = NULL;
660 droq->recv_buf_list[droq->read_idx].buffer =
661 NULL;
662
663 droq->read_idx = incr_index(droq->read_idx, 1,
664 droq->max_count);
665 droq->refill_count++;
666 } else {
667 nicbuf = octeon_fast_packet_alloc((u32)
668 info->length);
669 pkt_len = 0;
670 /* nicbuf allocation can fail. We'll handle it
671 * inside the loop.
672 */
673 while (pkt_len < info->length) {
674 int cpy_len, idx = droq->read_idx;
675
676 cpy_len = ((pkt_len + droq->buffer_size)
677 > info->length) ?
678 ((u32)info->length - pkt_len) :
679 droq->buffer_size;
680
681 if (nicbuf) {
682 octeon_fast_packet_next(droq,
683 nicbuf,
684 cpy_len,
685 idx);
686 buf = droq->recv_buf_list[idx].
687 buffer;
688 recv_buffer_fast_free(buf);
689 droq->recv_buf_list[idx].buffer
690 = NULL;
691 } else {
692 droq->stats.rx_alloc_failure++;
693 }
694
695 pkt_len += cpy_len;
696 droq->read_idx =
697 incr_index(droq->read_idx, 1,
698 droq->max_count);
699 droq->refill_count++;
700 }
701 }
702
703 if (nicbuf) {
704 if (droq->ops.fptr) {
705 droq->ops.fptr(oct->octeon_id,
706 nicbuf, pkt_len,
707 rh, &droq->napi,
708 droq->ops.farg);
709 } else {
710 recv_buffer_free(nicbuf);
711 }
712 }
713 }
714
715 if (droq->refill_count >= droq->refill_threshold) {
716 int desc_refilled = octeon_droq_refill(oct, droq);
717
718 /* Flush the droq descriptor data to memory to be sure
719 * that when we update the credits the data in memory
720 * is accurate.
721 */
722 wmb();
723 writel((desc_refilled), droq->pkts_credit_reg);
724 /* make sure mmio write completes */
725 mmiowb();
726 }
727
728 } /* for (each packet)... */
729
730 /* Increment refill_count by the number of buffers processed. */
731 droq->stats.pkts_received += pkt;
732 droq->stats.bytes_received += total_len;
733
734 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
735 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
736
737 droq->stats.dropped_toomany += (pkts_to_process - pkt);
738 return pkts_to_process;
739 }
740
741 return pkt;
742 }
743
744 int
745 octeon_droq_process_packets(struct octeon_device *oct,
746 struct octeon_droq *droq,
747 u32 budget)
748 {
749 u32 pkt_count = 0, pkts_processed = 0;
750 struct list_head *tmp, *tmp2;
751
752 /* Grab the droq lock */
753 spin_lock(&droq->lock);
754
755 octeon_droq_check_hw_for_pkts(droq);
756 pkt_count = atomic_read(&droq->pkts_pending);
757
758 if (!pkt_count) {
759 spin_unlock(&droq->lock);
760 return 0;
761 }
762
763 if (pkt_count > budget)
764 pkt_count = budget;
765
766 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
767
768 atomic_sub(pkts_processed, &droq->pkts_pending);
769
770 /* Release the spin lock */
771 spin_unlock(&droq->lock);
772
773 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
774 struct __dispatch *rdisp = (struct __dispatch *)tmp;
775
776 list_del(tmp);
777 rdisp->disp_fn(rdisp->rinfo,
778 octeon_get_dispatch_arg
779 (oct,
780 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
781 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
782 }
783
784 /* If there are packets pending. schedule tasklet again */
785 if (atomic_read(&droq->pkts_pending))
786 return 1;
787
788 return 0;
789 }
790
791 /**
792 * Utility function to poll for packets. check_hw_for_packets must be
793 * called before calling this routine.
794 */
795
796 static int
797 octeon_droq_process_poll_pkts(struct octeon_device *oct,
798 struct octeon_droq *droq, u32 budget)
799 {
800 struct list_head *tmp, *tmp2;
801 u32 pkts_available = 0, pkts_processed = 0;
802 u32 total_pkts_processed = 0;
803
804 if (budget > droq->max_count)
805 budget = droq->max_count;
806
807 spin_lock(&droq->lock);
808
809 while (total_pkts_processed < budget) {
810 octeon_droq_check_hw_for_pkts(droq);
811
812 pkts_available = min((budget - total_pkts_processed),
813 (u32)(atomic_read(&droq->pkts_pending)));
814
815 if (pkts_available == 0)
816 break;
817
818 pkts_processed =
819 octeon_droq_fast_process_packets(oct, droq,
820 pkts_available);
821
822 atomic_sub(pkts_processed, &droq->pkts_pending);
823
824 total_pkts_processed += pkts_processed;
825 }
826
827 spin_unlock(&droq->lock);
828
829 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
830 struct __dispatch *rdisp = (struct __dispatch *)tmp;
831
832 list_del(tmp);
833 rdisp->disp_fn(rdisp->rinfo,
834 octeon_get_dispatch_arg
835 (oct,
836 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
837 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
838 }
839
840 return total_pkts_processed;
841 }
842
843 int
844 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd,
845 u32 arg)
846 {
847 struct octeon_droq *droq;
848
849 droq = oct->droq[q_no];
850
851 if (cmd == POLL_EVENT_PROCESS_PKTS)
852 return octeon_droq_process_poll_pkts(oct, droq, arg);
853
854 if (cmd == POLL_EVENT_PENDING_PKTS) {
855 u32 pkt_cnt = atomic_read(&droq->pkts_pending);
856
857 return octeon_droq_process_packets(oct, droq, pkt_cnt);
858 }
859
860 if (cmd == POLL_EVENT_ENABLE_INTR) {
861 u32 value;
862 unsigned long flags;
863
864 /* Enable Pkt Interrupt */
865 switch (oct->chip_id) {
866 case OCTEON_CN66XX:
867 case OCTEON_CN68XX: {
868 struct octeon_cn6xxx *cn6xxx =
869 (struct octeon_cn6xxx *)oct->chip;
870 spin_lock_irqsave
871 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
872 value =
873 octeon_read_csr(oct,
874 CN6XXX_SLI_PKT_TIME_INT_ENB);
875 value |= (1 << q_no);
876 octeon_write_csr(oct,
877 CN6XXX_SLI_PKT_TIME_INT_ENB,
878 value);
879 value =
880 octeon_read_csr(oct,
881 CN6XXX_SLI_PKT_CNT_INT_ENB);
882 value |= (1 << q_no);
883 octeon_write_csr(oct,
884 CN6XXX_SLI_PKT_CNT_INT_ENB,
885 value);
886
887 /* don't bother flushing the enables */
888
889 spin_unlock_irqrestore
890 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
891 return 0;
892 }
893 break;
894 case OCTEON_CN23XX_PF_VID: {
895 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
896 }
897 break;
898
899 case OCTEON_CN23XX_VF_VID:
900 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
901 break;
902 }
903 return 0;
904 }
905
906 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd);
907 return -EINVAL;
908 }
909
910 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
911 struct octeon_droq_ops *ops)
912 {
913 struct octeon_droq *droq;
914 unsigned long flags;
915 struct octeon_config *oct_cfg = NULL;
916
917 oct_cfg = octeon_get_conf(oct);
918
919 if (!oct_cfg)
920 return -EINVAL;
921
922 if (!(ops)) {
923 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
924 __func__);
925 return -EINVAL;
926 }
927
928 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
929 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
930 __func__, q_no, (oct->num_oqs - 1));
931 return -EINVAL;
932 }
933
934 droq = oct->droq[q_no];
935
936 spin_lock_irqsave(&droq->lock, flags);
937
938 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
939
940 spin_unlock_irqrestore(&droq->lock, flags);
941
942 return 0;
943 }
944
945 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
946 {
947 unsigned long flags;
948 struct octeon_droq *droq;
949 struct octeon_config *oct_cfg = NULL;
950
951 oct_cfg = octeon_get_conf(oct);
952
953 if (!oct_cfg)
954 return -EINVAL;
955
956 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
957 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
958 __func__, q_no, oct->num_oqs - 1);
959 return -EINVAL;
960 }
961
962 droq = oct->droq[q_no];
963
964 if (!droq) {
965 dev_info(&oct->pci_dev->dev,
966 "Droq id (%d) not available.\n", q_no);
967 return 0;
968 }
969
970 spin_lock_irqsave(&droq->lock, flags);
971
972 droq->ops.fptr = NULL;
973 droq->ops.farg = NULL;
974 droq->ops.drop_on_max = 0;
975
976 spin_unlock_irqrestore(&droq->lock, flags);
977
978 return 0;
979 }
980
981 int octeon_create_droq(struct octeon_device *oct,
982 u32 q_no, u32 num_descs,
983 u32 desc_size, void *app_ctx)
984 {
985 struct octeon_droq *droq;
986 int numa_node = cpu_to_node(q_no % num_online_cpus());
987
988 if (oct->droq[q_no]) {
989 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
990 q_no);
991 return 1;
992 }
993
994 /* Allocate the DS for the new droq. */
995 droq = vmalloc_node(sizeof(*droq), numa_node);
996 if (!droq)
997 droq = vmalloc(sizeof(*droq));
998 if (!droq)
999 return -1;
1000
1001 memset(droq, 0, sizeof(struct octeon_droq));
1002
1003 /*Disable the pkt o/p for this Q */
1004 octeon_set_droq_pkt_op(oct, q_no, 0);
1005 oct->droq[q_no] = droq;
1006
1007 /* Initialize the Droq */
1008 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {
1009 vfree(oct->droq[q_no]);
1010 oct->droq[q_no] = NULL;
1011 return -1;
1012 }
1013
1014 oct->num_oqs++;
1015
1016 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
1017 oct->num_oqs);
1018
1019 /* Global Droq register settings */
1020
1021 /* As of now not required, as setting are done for all 32 Droqs at
1022 * the same time.
1023 */
1024 return 0;
1025 }
ubuntu-bionic-kernel mirror