]>
Commit | Line | Data |
---|---|---|
f21fb3ed | 1 | /********************************************************************** |
50579d3d RV |
2 | * Author: Cavium, Inc. |
3 | * | |
4 | * Contact: support@cavium.com | |
5 | * Please include "LiquidIO" in the subject. | |
6 | * | |
7 | * Copyright (c) 2003-2016 Cavium, Inc. | |
8 | * | |
9 | * This file is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License, Version 2, as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This file is distributed in the hope that it will be useful, but | |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty | |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or | |
16 | * NONINFRINGEMENT. See the GNU General Public License for more details. | |
17 | ***********************************************************************/ | |
f21fb3ed | 18 | #include <linux/pci.h> |
f21fb3ed | 19 | #include <linux/netdevice.h> |
5b173cf9 | 20 | #include <linux/vmalloc.h> |
f21fb3ed RV |
21 | #include "liquidio_common.h" |
22 | #include "octeon_droq.h" | |
23 | #include "octeon_iq.h" | |
24 | #include "response_manager.h" | |
25 | #include "octeon_device.h" | |
f21fb3ed RV |
26 | #include "octeon_main.h" |
27 | #include "octeon_network.h" | |
28 | #include "cn66xx_regs.h" | |
29 | #include "cn66xx_device.h" | |
5b823514 | 30 | #include "cn23xx_pf_device.h" |
9217c3cf | 31 | #include "cn23xx_vf_device.h" |
f21fb3ed | 32 | |
f21fb3ed RV |
33 | struct niclist { |
34 | struct list_head list; | |
35 | void *ptr; | |
36 | }; | |
37 | ||
38 | struct __dispatch { | |
39 | struct list_head list; | |
40 | struct octeon_recv_info *rinfo; | |
41 | octeon_dispatch_fn_t disp_fn; | |
42 | }; | |
43 | ||
44 | /** Get the argument that the user set when registering dispatch | |
45 | * function for a given opcode/subcode. | |
46 | * @param octeon_dev - the octeon device pointer. | |
47 | * @param opcode - the opcode for which the dispatch argument | |
48 | * is to be checked. | |
49 | * @param subcode - the subcode for which the dispatch argument | |
50 | * is to be checked. | |
51 | * @return Success: void * (argument to the dispatch function) | |
52 | * @return Failure: NULL | |
53 | * | |
54 | */ | |
55 | static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, | |
56 | u16 opcode, u16 subcode) | |
57 | { | |
58 | int idx; | |
59 | struct list_head *dispatch; | |
60 | void *fn_arg = NULL; | |
61 | u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); | |
62 | ||
63 | idx = combined_opcode & OCTEON_OPCODE_MASK; | |
64 | ||
65 | spin_lock_bh(&octeon_dev->dispatch.lock); | |
66 | ||
67 | if (octeon_dev->dispatch.count == 0) { | |
68 | spin_unlock_bh(&octeon_dev->dispatch.lock); | |
69 | return NULL; | |
70 | } | |
71 | ||
72 | if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { | |
73 | fn_arg = octeon_dev->dispatch.dlist[idx].arg; | |
74 | } else { | |
75 | list_for_each(dispatch, | |
76 | &octeon_dev->dispatch.dlist[idx].list) { | |
77 | if (((struct octeon_dispatch *)dispatch)->opcode == | |
78 | combined_opcode) { | |
79 | fn_arg = ((struct octeon_dispatch *) | |
80 | dispatch)->arg; | |
81 | break; | |
82 | } | |
83 | } | |
84 | } | |
85 | ||
86 | spin_unlock_bh(&octeon_dev->dispatch.lock); | |
87 | return fn_arg; | |
88 | } | |
89 | ||
cd8b1eb4 | 90 | /** Check for packets on Droq. This function should be called with lock held. |
a2c64b67 RV |
91 | * @param droq - Droq on which count is checked. |
92 | * @return Returns packet count. | |
93 | */ | |
a7d5a3dc | 94 | u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) |
f21fb3ed RV |
95 | { |
96 | u32 pkt_count = 0; | |
cd8b1eb4 | 97 | u32 last_count; |
f21fb3ed RV |
98 | |
99 | pkt_count = readl(droq->pkts_sent_reg); | |
f21fb3ed | 100 | |
cd8b1eb4 RV |
101 | last_count = pkt_count - droq->pkt_count; |
102 | droq->pkt_count = pkt_count; | |
103 | ||
104 | /* we shall write to cnts at napi irq enable or end of droq tasklet */ | |
105 | if (last_count) | |
106 | atomic_add(last_count, &droq->pkts_pending); | |
107 | ||
108 | return last_count; | |
f21fb3ed RV |
109 | } |
110 | ||
111 | static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) | |
112 | { | |
113 | u32 count = 0; | |
114 | ||
115 | /* max_empty_descs is the max. no. of descs that can have no buffers. | |
116 | * If the empty desc count goes beyond this value, we cannot safely | |
117 | * read in a 64K packet sent by Octeon | |
118 | * (64K is max pkt size from Octeon) | |
119 | */ | |
120 | droq->max_empty_descs = 0; | |
121 | ||
122 | do { | |
123 | droq->max_empty_descs++; | |
124 | count += droq->buffer_size; | |
125 | } while (count < (64 * 1024)); | |
126 | ||
127 | droq->max_empty_descs = droq->max_count - droq->max_empty_descs; | |
128 | } | |
129 | ||
130 | static void octeon_droq_reset_indices(struct octeon_droq *droq) | |
131 | { | |
132 | droq->read_idx = 0; | |
133 | droq->write_idx = 0; | |
134 | droq->refill_idx = 0; | |
135 | droq->refill_count = 0; | |
136 | atomic_set(&droq->pkts_pending, 0); | |
137 | } | |
138 | ||
139 | static void | |
140 | octeon_droq_destroy_ring_buffers(struct octeon_device *oct, | |
141 | struct octeon_droq *droq) | |
142 | { | |
143 | u32 i; | |
cabeb13b | 144 | struct octeon_skb_page_info *pg_info; |
f21fb3ed RV |
145 | |
146 | for (i = 0; i < droq->max_count; i++) { | |
cabeb13b RV |
147 | pg_info = &droq->recv_buf_list[i].pg_info; |
148 | ||
149 | if (pg_info->dma) | |
150 | lio_unmap_ring(oct->pci_dev, | |
151 | (u64)pg_info->dma); | |
152 | pg_info->dma = 0; | |
153 | ||
154 | if (pg_info->page) | |
155 | recv_buffer_destroy(droq->recv_buf_list[i].buffer, | |
156 | pg_info); | |
157 | ||
cabeb13b | 158 | droq->recv_buf_list[i].buffer = NULL; |
f21fb3ed RV |
159 | } |
160 | ||
161 | octeon_droq_reset_indices(droq); | |
162 | } | |
163 | ||
164 | static int | |
165 | octeon_droq_setup_ring_buffers(struct octeon_device *oct, | |
166 | struct octeon_droq *droq) | |
167 | { | |
168 | u32 i; | |
169 | void *buf; | |
170 | struct octeon_droq_desc *desc_ring = droq->desc_ring; | |
171 | ||
172 | for (i = 0; i < droq->max_count; i++) { | |
cabeb13b | 173 | buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); |
f21fb3ed RV |
174 | |
175 | if (!buf) { | |
176 | dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", | |
177 | __func__); | |
cabeb13b | 178 | droq->stats.rx_alloc_failure++; |
f21fb3ed RV |
179 | return -ENOMEM; |
180 | } | |
181 | ||
182 | droq->recv_buf_list[i].buffer = buf; | |
183 | droq->recv_buf_list[i].data = get_rbd(buf); | |
f21fb3ed RV |
184 | droq->info_list[i].length = 0; |
185 | ||
186 | /* map ring buffers into memory */ | |
187 | desc_ring[i].info_ptr = lio_map_ring_info(droq, i); | |
188 | desc_ring[i].buffer_ptr = | |
cabeb13b | 189 | lio_map_ring(droq->recv_buf_list[i].buffer); |
f21fb3ed RV |
190 | } |
191 | ||
192 | octeon_droq_reset_indices(droq); | |
193 | ||
194 | octeon_droq_compute_max_packet_bufs(droq); | |
195 | ||
196 | return 0; | |
197 | } | |
198 | ||
199 | int octeon_delete_droq(struct octeon_device *oct, u32 q_no) | |
200 | { | |
201 | struct octeon_droq *droq = oct->droq[q_no]; | |
202 | ||
203 | dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); | |
204 | ||
205 | octeon_droq_destroy_ring_buffers(oct, droq); | |
9686f310 | 206 | vfree(droq->recv_buf_list); |
f21fb3ed RV |
207 | |
208 | if (droq->info_base_addr) | |
67e303e0 | 209 | lio_free_info_buffer(oct, droq); |
f21fb3ed RV |
210 | |
211 | if (droq->desc_ring) | |
212 | lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), | |
213 | droq->desc_ring, droq->desc_ring_dma); | |
214 | ||
215 | memset(droq, 0, OCT_DROQ_SIZE); | |
216 | ||
217 | return 0; | |
218 | } | |
219 | ||
220 | int octeon_init_droq(struct octeon_device *oct, | |
221 | u32 q_no, | |
222 | u32 num_descs, | |
223 | u32 desc_size, | |
224 | void *app_ctx) | |
225 | { | |
226 | struct octeon_droq *droq; | |
227 | u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; | |
228 | u32 c_pkts_per_intr = 0, c_refill_threshold = 0; | |
96ae48b7 RV |
229 | int orig_node = dev_to_node(&oct->pci_dev->dev); |
230 | int numa_node = cpu_to_node(q_no % num_online_cpus()); | |
f21fb3ed RV |
231 | |
232 | dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); | |
233 | ||
234 | droq = oct->droq[q_no]; | |
235 | memset(droq, 0, OCT_DROQ_SIZE); | |
236 | ||
237 | droq->oct_dev = oct; | |
238 | droq->q_no = q_no; | |
239 | if (app_ctx) | |
240 | droq->app_ctx = app_ctx; | |
241 | else | |
242 | droq->app_ctx = (void *)(size_t)q_no; | |
243 | ||
244 | c_num_descs = num_descs; | |
245 | c_buf_size = desc_size; | |
246 | if (OCTEON_CN6XXX(oct)) { | |
97a25326 | 247 | struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); |
f21fb3ed RV |
248 | |
249 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); | |
96ae48b7 RV |
250 | c_refill_threshold = |
251 | (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); | |
5b823514 | 252 | } else if (OCTEON_CN23XX_PF(oct)) { |
97a25326 | 253 | struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); |
5b823514 | 254 | |
9217c3cf RV |
255 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); |
256 | c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); | |
257 | } else if (OCTEON_CN23XX_VF(oct)) { | |
258 | struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); | |
259 | ||
5b823514 RV |
260 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); |
261 | c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); | |
96ae48b7 RV |
262 | } else { |
263 | return 1; | |
f21fb3ed RV |
264 | } |
265 | ||
266 | droq->max_count = c_num_descs; | |
267 | droq->buffer_size = c_buf_size; | |
268 | ||
269 | desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; | |
96ae48b7 | 270 | set_dev_node(&oct->pci_dev->dev, numa_node); |
f21fb3ed RV |
271 | droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, |
272 | (dma_addr_t *)&droq->desc_ring_dma); | |
96ae48b7 RV |
273 | set_dev_node(&oct->pci_dev->dev, orig_node); |
274 | if (!droq->desc_ring) | |
275 | droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, | |
276 | (dma_addr_t *)&droq->desc_ring_dma); | |
f21fb3ed RV |
277 | |
278 | if (!droq->desc_ring) { | |
279 | dev_err(&oct->pci_dev->dev, | |
280 | "Output queue %d ring alloc failed\n", q_no); | |
281 | return 1; | |
282 | } | |
283 | ||
284 | dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", | |
285 | q_no, droq->desc_ring, droq->desc_ring_dma); | |
286 | dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, | |
287 | droq->max_count); | |
288 | ||
67e303e0 | 289 | droq->info_list = lio_alloc_info_buffer(oct, droq); |
f21fb3ed RV |
290 | if (!droq->info_list) { |
291 | dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n"); | |
292 | lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), | |
293 | droq->desc_ring, droq->desc_ring_dma); | |
294 | return 1; | |
295 | } | |
296 | ||
297 | droq->recv_buf_list = (struct octeon_recv_buffer *) | |
96ae48b7 RV |
298 | vmalloc_node(droq->max_count * |
299 | OCT_DROQ_RECVBUF_SIZE, | |
300 | numa_node); | |
301 | if (!droq->recv_buf_list) | |
302 | droq->recv_buf_list = (struct octeon_recv_buffer *) | |
303 | vmalloc(droq->max_count * | |
f21fb3ed RV |
304 | OCT_DROQ_RECVBUF_SIZE); |
305 | if (!droq->recv_buf_list) { | |
306 | dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); | |
307 | goto init_droq_fail; | |
308 | } | |
309 | ||
310 | if (octeon_droq_setup_ring_buffers(oct, droq)) | |
311 | goto init_droq_fail; | |
312 | ||
313 | droq->pkts_per_intr = c_pkts_per_intr; | |
314 | droq->refill_threshold = c_refill_threshold; | |
315 | ||
316 | dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", | |
317 | droq->max_empty_descs); | |
318 | ||
319 | spin_lock_init(&droq->lock); | |
320 | ||
321 | INIT_LIST_HEAD(&droq->dispatch_list); | |
322 | ||
323 | /* For 56xx Pass1, this function won't be called, so no checks. */ | |
324 | oct->fn_list.setup_oq_regs(oct, q_no); | |
325 | ||
763185a3 | 326 | oct->io_qmask.oq |= BIT_ULL(q_no); |
f21fb3ed RV |
327 | |
328 | return 0; | |
329 | ||
330 | init_droq_fail: | |
331 | octeon_delete_droq(oct, q_no); | |
332 | return 1; | |
333 | } | |
334 | ||
335 | /* octeon_create_recv_info | |
336 | * Parameters: | |
337 | * octeon_dev - pointer to the octeon device structure | |
338 | * droq - droq in which the packet arrived. | |
339 | * buf_cnt - no. of buffers used by the packet. | |
340 | * idx - index in the descriptor for the first buffer in the packet. | |
341 | * Description: | |
342 | * Allocates a recv_info_t and copies the buffer addresses for packet data | |
343 | * into the recv_pkt space which starts at an 8B offset from recv_info_t. | |
344 | * Flags the descriptors for refill later. If available descriptors go | |
345 | * below the threshold to receive a 64K pkt, new buffers are first allocated | |
346 | * before the recv_pkt_t is created. | |
347 | * This routine will be called in interrupt context. | |
348 | * Returns: | |
349 | * Success: Pointer to recv_info_t | |
350 | * Failure: NULL. | |
351 | * Locks: | |
352 | * The droq->lock is held when this routine is called. | |
353 | */ | |
354 | static inline struct octeon_recv_info *octeon_create_recv_info( | |
355 | struct octeon_device *octeon_dev, | |
356 | struct octeon_droq *droq, | |
357 | u32 buf_cnt, | |
358 | u32 idx) | |
359 | { | |
360 | struct octeon_droq_info *info; | |
361 | struct octeon_recv_pkt *recv_pkt; | |
362 | struct octeon_recv_info *recv_info; | |
363 | u32 i, bytes_left; | |
cabeb13b | 364 | struct octeon_skb_page_info *pg_info; |
f21fb3ed RV |
365 | |
366 | info = &droq->info_list[idx]; | |
367 | ||
368 | recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); | |
369 | if (!recv_info) | |
370 | return NULL; | |
371 | ||
372 | recv_pkt = recv_info->recv_pkt; | |
373 | recv_pkt->rh = info->rh; | |
374 | recv_pkt->length = (u32)info->length; | |
375 | recv_pkt->buffer_count = (u16)buf_cnt; | |
376 | recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; | |
377 | ||
378 | i = 0; | |
379 | bytes_left = (u32)info->length; | |
380 | ||
381 | while (buf_cnt) { | |
cabeb13b RV |
382 | { |
383 | pg_info = &droq->recv_buf_list[idx].pg_info; | |
384 | ||
385 | lio_unmap_ring(octeon_dev->pci_dev, | |
386 | (u64)pg_info->dma); | |
387 | pg_info->page = NULL; | |
388 | pg_info->dma = 0; | |
389 | } | |
f21fb3ed RV |
390 | |
391 | recv_pkt->buffer_size[i] = | |
392 | (bytes_left >= | |
393 | droq->buffer_size) ? droq->buffer_size : bytes_left; | |
394 | ||
395 | recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; | |
396 | droq->recv_buf_list[idx].buffer = NULL; | |
397 | ||
97a25326 | 398 | idx = incr_index(idx, 1, droq->max_count); |
f21fb3ed RV |
399 | bytes_left -= droq->buffer_size; |
400 | i++; | |
401 | buf_cnt--; | |
402 | } | |
403 | ||
404 | return recv_info; | |
405 | } | |
406 | ||
407 | /* If we were not able to refill all buffers, try to move around | |
408 | * the buffers that were not dispatched. | |
409 | */ | |
410 | static inline u32 | |
411 | octeon_droq_refill_pullup_descs(struct octeon_droq *droq, | |
412 | struct octeon_droq_desc *desc_ring) | |
413 | { | |
414 | u32 desc_refilled = 0; | |
415 | ||
416 | u32 refill_index = droq->refill_idx; | |
417 | ||
418 | while (refill_index != droq->read_idx) { | |
419 | if (droq->recv_buf_list[refill_index].buffer) { | |
420 | droq->recv_buf_list[droq->refill_idx].buffer = | |
421 | droq->recv_buf_list[refill_index].buffer; | |
422 | droq->recv_buf_list[droq->refill_idx].data = | |
423 | droq->recv_buf_list[refill_index].data; | |
424 | desc_ring[droq->refill_idx].buffer_ptr = | |
425 | desc_ring[refill_index].buffer_ptr; | |
426 | droq->recv_buf_list[refill_index].buffer = NULL; | |
427 | desc_ring[refill_index].buffer_ptr = 0; | |
428 | do { | |
97a25326 RV |
429 | droq->refill_idx = incr_index(droq->refill_idx, |
430 | 1, | |
431 | droq->max_count); | |
f21fb3ed RV |
432 | desc_refilled++; |
433 | droq->refill_count--; | |
434 | } while (droq->recv_buf_list[droq->refill_idx]. | |
435 | buffer); | |
436 | } | |
97a25326 | 437 | refill_index = incr_index(refill_index, 1, droq->max_count); |
f21fb3ed RV |
438 | } /* while */ |
439 | return desc_refilled; | |
440 | } | |
441 | ||
442 | /* octeon_droq_refill | |
443 | * Parameters: | |
444 | * droq - droq in which descriptors require new buffers. | |
445 | * Description: | |
446 | * Called during normal DROQ processing in interrupt mode or by the poll | |
447 | * thread to refill the descriptors from which buffers were dispatched | |
448 | * to upper layers. Attempts to allocate new buffers. If that fails, moves | |
449 | * up buffers (that were not dispatched) to form a contiguous ring. | |
450 | * Returns: | |
451 | * No of descriptors refilled. | |
452 | * Locks: | |
453 | * This routine is called with droq->lock held. | |
454 | */ | |
455 | static u32 | |
456 | octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) | |
457 | { | |
458 | struct octeon_droq_desc *desc_ring; | |
459 | void *buf = NULL; | |
460 | u8 *data; | |
461 | u32 desc_refilled = 0; | |
cabeb13b | 462 | struct octeon_skb_page_info *pg_info; |
f21fb3ed RV |
463 | |
464 | desc_ring = droq->desc_ring; | |
465 | ||
466 | while (droq->refill_count && (desc_refilled < droq->max_count)) { | |
467 | /* If a valid buffer exists (happens if there is no dispatch), | |
468 | * reuse | |
469 | * the buffer, else allocate. | |
470 | */ | |
471 | if (!droq->recv_buf_list[droq->refill_idx].buffer) { | |
cabeb13b RV |
472 | pg_info = |
473 | &droq->recv_buf_list[droq->refill_idx].pg_info; | |
474 | /* Either recycle the existing pages or go for | |
475 | * new page alloc | |
476 | */ | |
477 | if (pg_info->page) | |
478 | buf = recv_buffer_reuse(octeon_dev, pg_info); | |
479 | else | |
480 | buf = recv_buffer_alloc(octeon_dev, pg_info); | |
f21fb3ed RV |
481 | /* If a buffer could not be allocated, no point in |
482 | * continuing | |
483 | */ | |
cabeb13b RV |
484 | if (!buf) { |
485 | droq->stats.rx_alloc_failure++; | |
f21fb3ed | 486 | break; |
cabeb13b | 487 | } |
f21fb3ed RV |
488 | droq->recv_buf_list[droq->refill_idx].buffer = |
489 | buf; | |
490 | data = get_rbd(buf); | |
491 | } else { | |
492 | data = get_rbd(droq->recv_buf_list | |
493 | [droq->refill_idx].buffer); | |
494 | } | |
495 | ||
496 | droq->recv_buf_list[droq->refill_idx].data = data; | |
497 | ||
498 | desc_ring[droq->refill_idx].buffer_ptr = | |
cabeb13b RV |
499 | lio_map_ring(droq->recv_buf_list[droq-> |
500 | refill_idx].buffer); | |
f21fb3ed RV |
501 | /* Reset any previous values in the length field. */ |
502 | droq->info_list[droq->refill_idx].length = 0; | |
503 | ||
97a25326 RV |
504 | droq->refill_idx = incr_index(droq->refill_idx, 1, |
505 | droq->max_count); | |
f21fb3ed RV |
506 | desc_refilled++; |
507 | droq->refill_count--; | |
508 | } | |
509 | ||
510 | if (droq->refill_count) | |
511 | desc_refilled += | |
512 | octeon_droq_refill_pullup_descs(droq, desc_ring); | |
513 | ||
514 | /* if droq->refill_count | |
515 | * The refill count would not change in pass two. We only moved buffers | |
516 | * to close the gap in the ring, but we would still have the same no. of | |
517 | * buffers to refill. | |
518 | */ | |
519 | return desc_refilled; | |
520 | } | |
521 | ||
522 | static inline u32 | |
523 | octeon_droq_get_bufcount(u32 buf_size, u32 total_len) | |
524 | { | |
525 | u32 buf_cnt = 0; | |
526 | ||
527 | while (total_len > (buf_size * buf_cnt)) | |
528 | buf_cnt++; | |
529 | return buf_cnt; | |
530 | } | |
531 | ||
532 | static int | |
533 | octeon_droq_dispatch_pkt(struct octeon_device *oct, | |
534 | struct octeon_droq *droq, | |
535 | union octeon_rh *rh, | |
536 | struct octeon_droq_info *info) | |
537 | { | |
538 | u32 cnt; | |
539 | octeon_dispatch_fn_t disp_fn; | |
540 | struct octeon_recv_info *rinfo; | |
541 | ||
542 | cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); | |
543 | ||
544 | disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, | |
545 | (u16)rh->r.subcode); | |
546 | if (disp_fn) { | |
547 | rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); | |
548 | if (rinfo) { | |
549 | struct __dispatch *rdisp = rinfo->rsvd; | |
550 | ||
551 | rdisp->rinfo = rinfo; | |
552 | rdisp->disp_fn = disp_fn; | |
553 | rinfo->recv_pkt->rh = *rh; | |
554 | list_add_tail(&rdisp->list, | |
555 | &droq->dispatch_list); | |
556 | } else { | |
557 | droq->stats.dropped_nomem++; | |
558 | } | |
559 | } else { | |
a2c64b67 RV |
560 | dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", |
561 | (unsigned int)rh->r.opcode, | |
562 | (unsigned int)rh->r.subcode); | |
f21fb3ed | 563 | droq->stats.dropped_nodispatch++; |
9ded1a51 | 564 | } |
f21fb3ed RV |
565 | |
566 | return cnt; | |
567 | } | |
568 | ||
569 | static inline void octeon_droq_drop_packets(struct octeon_device *oct, | |
570 | struct octeon_droq *droq, | |
571 | u32 cnt) | |
572 | { | |
573 | u32 i = 0, buf_cnt; | |
574 | struct octeon_droq_info *info; | |
575 | ||
576 | for (i = 0; i < cnt; i++) { | |
577 | info = &droq->info_list[droq->read_idx]; | |
578 | octeon_swap_8B_data((u64 *)info, 2); | |
579 | ||
580 | if (info->length) { | |
581 | info->length -= OCT_RH_SIZE; | |
582 | droq->stats.bytes_received += info->length; | |
583 | buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, | |
584 | (u32)info->length); | |
585 | } else { | |
586 | dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); | |
587 | buf_cnt = 1; | |
588 | } | |
589 | ||
97a25326 RV |
590 | droq->read_idx = incr_index(droq->read_idx, buf_cnt, |
591 | droq->max_count); | |
f21fb3ed RV |
592 | droq->refill_count += buf_cnt; |
593 | } | |
594 | } | |
595 | ||
596 | static u32 | |
597 | octeon_droq_fast_process_packets(struct octeon_device *oct, | |
598 | struct octeon_droq *droq, | |
599 | u32 pkts_to_process) | |
600 | { | |
601 | struct octeon_droq_info *info; | |
602 | union octeon_rh *rh; | |
603 | u32 pkt, total_len = 0, pkt_count; | |
604 | ||
605 | pkt_count = pkts_to_process; | |
606 | ||
607 | for (pkt = 0; pkt < pkt_count; pkt++) { | |
608 | u32 pkt_len = 0; | |
609 | struct sk_buff *nicbuf = NULL; | |
cabeb13b RV |
610 | struct octeon_skb_page_info *pg_info; |
611 | void *buf; | |
f21fb3ed RV |
612 | |
613 | info = &droq->info_list[droq->read_idx]; | |
614 | octeon_swap_8B_data((u64 *)info, 2); | |
615 | ||
616 | if (!info->length) { | |
617 | dev_err(&oct->pci_dev->dev, | |
618 | "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", | |
619 | droq->q_no, droq->read_idx, pkt_count); | |
620 | print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, | |
621 | (u8 *)info, | |
622 | OCT_DROQ_INFO_SIZE); | |
623 | break; | |
624 | } | |
625 | ||
626 | /* Len of resp hdr in included in the received data len. */ | |
627 | info->length -= OCT_RH_SIZE; | |
628 | rh = &info->rh; | |
629 | ||
630 | total_len += (u32)info->length; | |
97a25326 | 631 | if (opcode_slow_path(rh)) { |
f21fb3ed RV |
632 | u32 buf_cnt; |
633 | ||
634 | buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); | |
97a25326 RV |
635 | droq->read_idx = incr_index(droq->read_idx, |
636 | buf_cnt, droq->max_count); | |
f21fb3ed RV |
637 | droq->refill_count += buf_cnt; |
638 | } else { | |
639 | if (info->length <= droq->buffer_size) { | |
f21fb3ed RV |
640 | pkt_len = (u32)info->length; |
641 | nicbuf = droq->recv_buf_list[ | |
642 | droq->read_idx].buffer; | |
cabeb13b RV |
643 | pg_info = &droq->recv_buf_list[ |
644 | droq->read_idx].pg_info; | |
645 | if (recv_buffer_recycle(oct, pg_info)) | |
646 | pg_info->page = NULL; | |
f21fb3ed RV |
647 | droq->recv_buf_list[droq->read_idx].buffer = |
648 | NULL; | |
a2c64b67 | 649 | |
97a25326 RV |
650 | droq->read_idx = incr_index(droq->read_idx, 1, |
651 | droq->max_count); | |
f21fb3ed RV |
652 | droq->refill_count++; |
653 | } else { | |
cabeb13b | 654 | nicbuf = octeon_fast_packet_alloc((u32) |
f21fb3ed RV |
655 | info->length); |
656 | pkt_len = 0; | |
657 | /* nicbuf allocation can fail. We'll handle it | |
658 | * inside the loop. | |
659 | */ | |
660 | while (pkt_len < info->length) { | |
cabeb13b | 661 | int cpy_len, idx = droq->read_idx; |
f21fb3ed | 662 | |
cabeb13b RV |
663 | cpy_len = ((pkt_len + droq->buffer_size) |
664 | > info->length) ? | |
f21fb3ed RV |
665 | ((u32)info->length - pkt_len) : |
666 | droq->buffer_size; | |
667 | ||
668 | if (nicbuf) { | |
f21fb3ed RV |
669 | octeon_fast_packet_next(droq, |
670 | nicbuf, | |
671 | cpy_len, | |
cabeb13b RV |
672 | idx); |
673 | buf = droq->recv_buf_list[idx]. | |
674 | buffer; | |
675 | recv_buffer_fast_free(buf); | |
676 | droq->recv_buf_list[idx].buffer | |
677 | = NULL; | |
678 | } else { | |
679 | droq->stats.rx_alloc_failure++; | |
f21fb3ed RV |
680 | } |
681 | ||
682 | pkt_len += cpy_len; | |
97a25326 RV |
683 | droq->read_idx = |
684 | incr_index(droq->read_idx, 1, | |
685 | droq->max_count); | |
f21fb3ed RV |
686 | droq->refill_count++; |
687 | } | |
688 | } | |
689 | ||
690 | if (nicbuf) { | |
cabeb13b | 691 | if (droq->ops.fptr) { |
f21fb3ed | 692 | droq->ops.fptr(oct->octeon_id, |
cabeb13b | 693 | nicbuf, pkt_len, |
0cece6c5 RV |
694 | rh, &droq->napi, |
695 | droq->ops.farg); | |
cabeb13b | 696 | } else { |
f21fb3ed | 697 | recv_buffer_free(nicbuf); |
cabeb13b | 698 | } |
f21fb3ed RV |
699 | } |
700 | } | |
701 | ||
702 | if (droq->refill_count >= droq->refill_threshold) { | |
703 | int desc_refilled = octeon_droq_refill(oct, droq); | |
704 | ||
705 | /* Flush the droq descriptor data to memory to be sure | |
cabeb13b RV |
706 | * that when we update the credits the data in memory |
707 | * is accurate. | |
708 | */ | |
f21fb3ed RV |
709 | wmb(); |
710 | writel((desc_refilled), droq->pkts_credit_reg); | |
711 | /* make sure mmio write completes */ | |
712 | mmiowb(); | |
713 | } | |
714 | ||
cabeb13b | 715 | } /* for (each packet)... */ |
f21fb3ed RV |
716 | |
717 | /* Increment refill_count by the number of buffers processed. */ | |
718 | droq->stats.pkts_received += pkt; | |
719 | droq->stats.bytes_received += total_len; | |
720 | ||
721 | if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { | |
722 | octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); | |
723 | ||
724 | droq->stats.dropped_toomany += (pkts_to_process - pkt); | |
725 | return pkts_to_process; | |
726 | } | |
727 | ||
728 | return pkt; | |
729 | } | |
730 | ||
731 | int | |
732 | octeon_droq_process_packets(struct octeon_device *oct, | |
733 | struct octeon_droq *droq, | |
734 | u32 budget) | |
735 | { | |
736 | u32 pkt_count = 0, pkts_processed = 0; | |
737 | struct list_head *tmp, *tmp2; | |
738 | ||
cd8b1eb4 RV |
739 | /* Grab the droq lock */ |
740 | spin_lock(&droq->lock); | |
741 | ||
742 | octeon_droq_check_hw_for_pkts(droq); | |
f21fb3ed | 743 | pkt_count = atomic_read(&droq->pkts_pending); |
cd8b1eb4 RV |
744 | |
745 | if (!pkt_count) { | |
746 | spin_unlock(&droq->lock); | |
f21fb3ed | 747 | return 0; |
cd8b1eb4 | 748 | } |
f21fb3ed RV |
749 | |
750 | if (pkt_count > budget) | |
751 | pkt_count = budget; | |
752 | ||
f21fb3ed RV |
753 | pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); |
754 | ||
755 | atomic_sub(pkts_processed, &droq->pkts_pending); | |
756 | ||
757 | /* Release the spin lock */ | |
758 | spin_unlock(&droq->lock); | |
759 | ||
760 | list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { | |
761 | struct __dispatch *rdisp = (struct __dispatch *)tmp; | |
762 | ||
763 | list_del(tmp); | |
764 | rdisp->disp_fn(rdisp->rinfo, | |
765 | octeon_get_dispatch_arg | |
766 | (oct, | |
767 | (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, | |
768 | (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); | |
769 | } | |
770 | ||
771 | /* If there are packets pending. schedule tasklet again */ | |
772 | if (atomic_read(&droq->pkts_pending)) | |
773 | return 1; | |
774 | ||
775 | return 0; | |
776 | } | |
777 | ||
778 | /** | |
779 | * Utility function to poll for packets. check_hw_for_packets must be | |
780 | * called before calling this routine. | |
781 | */ | |
782 | ||
783 | static int | |
784 | octeon_droq_process_poll_pkts(struct octeon_device *oct, | |
785 | struct octeon_droq *droq, u32 budget) | |
786 | { | |
787 | struct list_head *tmp, *tmp2; | |
788 | u32 pkts_available = 0, pkts_processed = 0; | |
789 | u32 total_pkts_processed = 0; | |
790 | ||
791 | if (budget > droq->max_count) | |
792 | budget = droq->max_count; | |
793 | ||
794 | spin_lock(&droq->lock); | |
795 | ||
796 | while (total_pkts_processed < budget) { | |
cd8b1eb4 RV |
797 | octeon_droq_check_hw_for_pkts(droq); |
798 | ||
97a25326 RV |
799 | pkts_available = min((budget - total_pkts_processed), |
800 | (u32)(atomic_read(&droq->pkts_pending))); | |
f21fb3ed RV |
801 | |
802 | if (pkts_available == 0) | |
803 | break; | |
804 | ||
805 | pkts_processed = | |
806 | octeon_droq_fast_process_packets(oct, droq, | |
807 | pkts_available); | |
808 | ||
809 | atomic_sub(pkts_processed, &droq->pkts_pending); | |
810 | ||
811 | total_pkts_processed += pkts_processed; | |
f21fb3ed RV |
812 | } |
813 | ||
814 | spin_unlock(&droq->lock); | |
815 | ||
816 | list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { | |
817 | struct __dispatch *rdisp = (struct __dispatch *)tmp; | |
818 | ||
819 | list_del(tmp); | |
820 | rdisp->disp_fn(rdisp->rinfo, | |
821 | octeon_get_dispatch_arg | |
822 | (oct, | |
823 | (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, | |
824 | (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); | |
825 | } | |
826 | ||
827 | return total_pkts_processed; | |
828 | } | |
829 | ||
830 | int | |
831 | octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, | |
832 | u32 arg) | |
833 | { | |
834 | struct octeon_droq *droq; | |
f21fb3ed RV |
835 | |
836 | droq = oct->droq[q_no]; | |
837 | ||
838 | if (cmd == POLL_EVENT_PROCESS_PKTS) | |
839 | return octeon_droq_process_poll_pkts(oct, droq, arg); | |
840 | ||
841 | if (cmd == POLL_EVENT_PENDING_PKTS) { | |
842 | u32 pkt_cnt = atomic_read(&droq->pkts_pending); | |
843 | ||
844 | return octeon_droq_process_packets(oct, droq, pkt_cnt); | |
845 | } | |
846 | ||
847 | if (cmd == POLL_EVENT_ENABLE_INTR) { | |
848 | u32 value; | |
849 | unsigned long flags; | |
850 | ||
851 | /* Enable Pkt Interrupt */ | |
852 | switch (oct->chip_id) { | |
853 | case OCTEON_CN66XX: | |
854 | case OCTEON_CN68XX: { | |
855 | struct octeon_cn6xxx *cn6xxx = | |
856 | (struct octeon_cn6xxx *)oct->chip; | |
857 | spin_lock_irqsave | |
858 | (&cn6xxx->lock_for_droq_int_enb_reg, flags); | |
859 | value = | |
860 | octeon_read_csr(oct, | |
861 | CN6XXX_SLI_PKT_TIME_INT_ENB); | |
862 | value |= (1 << q_no); | |
863 | octeon_write_csr(oct, | |
864 | CN6XXX_SLI_PKT_TIME_INT_ENB, | |
865 | value); | |
866 | value = | |
867 | octeon_read_csr(oct, | |
868 | CN6XXX_SLI_PKT_CNT_INT_ENB); | |
869 | value |= (1 << q_no); | |
870 | octeon_write_csr(oct, | |
871 | CN6XXX_SLI_PKT_CNT_INT_ENB, | |
872 | value); | |
873 | ||
874 | /* don't bother flushing the enables */ | |
875 | ||
876 | spin_unlock_irqrestore | |
877 | (&cn6xxx->lock_for_droq_int_enb_reg, flags); | |
878 | return 0; | |
879 | } | |
880 | break; | |
9ded1a51 RV |
881 | case OCTEON_CN23XX_PF_VID: { |
882 | lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); | |
883 | } | |
884 | break; | |
9217c3cf RV |
885 | |
886 | case OCTEON_CN23XX_VF_VID: | |
887 | lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); | |
888 | break; | |
f21fb3ed | 889 | } |
f21fb3ed RV |
890 | return 0; |
891 | } | |
892 | ||
893 | dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); | |
894 | return -EINVAL; | |
895 | } | |
896 | ||
897 | int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, | |
898 | struct octeon_droq_ops *ops) | |
899 | { | |
900 | struct octeon_droq *droq; | |
901 | unsigned long flags; | |
902 | struct octeon_config *oct_cfg = NULL; | |
903 | ||
904 | oct_cfg = octeon_get_conf(oct); | |
905 | ||
906 | if (!oct_cfg) | |
907 | return -EINVAL; | |
908 | ||
909 | if (!(ops)) { | |
910 | dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", | |
911 | __func__); | |
912 | return -EINVAL; | |
913 | } | |
914 | ||
915 | if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { | |
916 | dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", | |
917 | __func__, q_no, (oct->num_oqs - 1)); | |
918 | return -EINVAL; | |
919 | } | |
920 | ||
921 | droq = oct->droq[q_no]; | |
922 | ||
923 | spin_lock_irqsave(&droq->lock, flags); | |
924 | ||
925 | memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); | |
926 | ||
927 | spin_unlock_irqrestore(&droq->lock, flags); | |
928 | ||
929 | return 0; | |
930 | } | |
931 | ||
932 | int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) | |
933 | { | |
934 | unsigned long flags; | |
935 | struct octeon_droq *droq; | |
936 | struct octeon_config *oct_cfg = NULL; | |
937 | ||
938 | oct_cfg = octeon_get_conf(oct); | |
939 | ||
940 | if (!oct_cfg) | |
941 | return -EINVAL; | |
942 | ||
943 | if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { | |
944 | dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", | |
945 | __func__, q_no, oct->num_oqs - 1); | |
946 | return -EINVAL; | |
947 | } | |
948 | ||
949 | droq = oct->droq[q_no]; | |
950 | ||
951 | if (!droq) { | |
952 | dev_info(&oct->pci_dev->dev, | |
953 | "Droq id (%d) not available.\n", q_no); | |
954 | return 0; | |
955 | } | |
956 | ||
957 | spin_lock_irqsave(&droq->lock, flags); | |
958 | ||
959 | droq->ops.fptr = NULL; | |
0cece6c5 | 960 | droq->ops.farg = NULL; |
f21fb3ed RV |
961 | droq->ops.drop_on_max = 0; |
962 | ||
963 | spin_unlock_irqrestore(&droq->lock, flags); | |
964 | ||
965 | return 0; | |
966 | } | |
967 | ||
968 | int octeon_create_droq(struct octeon_device *oct, | |
969 | u32 q_no, u32 num_descs, | |
970 | u32 desc_size, void *app_ctx) | |
971 | { | |
972 | struct octeon_droq *droq; | |
96ae48b7 | 973 | int numa_node = cpu_to_node(q_no % num_online_cpus()); |
f21fb3ed RV |
974 | |
975 | if (oct->droq[q_no]) { | |
976 | dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", | |
977 | q_no); | |
978 | return 1; | |
979 | } | |
980 | ||
981 | /* Allocate the DS for the new droq. */ | |
96ae48b7 RV |
982 | droq = vmalloc_node(sizeof(*droq), numa_node); |
983 | if (!droq) | |
984 | droq = vmalloc(sizeof(*droq)); | |
f21fb3ed | 985 | if (!droq) |
515e752d RV |
986 | return -1; |
987 | ||
f21fb3ed RV |
988 | memset(droq, 0, sizeof(struct octeon_droq)); |
989 | ||
990 | /*Disable the pkt o/p for this Q */ | |
991 | octeon_set_droq_pkt_op(oct, q_no, 0); | |
992 | oct->droq[q_no] = droq; | |
993 | ||
994 | /* Initialize the Droq */ | |
515e752d RV |
995 | if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { |
996 | vfree(oct->droq[q_no]); | |
997 | oct->droq[q_no] = NULL; | |
998 | return -1; | |
999 | } | |
f21fb3ed RV |
1000 | |
1001 | oct->num_oqs++; | |
1002 | ||
1003 | dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, | |
1004 | oct->num_oqs); | |
1005 | ||
1006 | /* Global Droq register settings */ | |
1007 | ||
1008 | /* As of now not required, as setting are done for all 32 Droqs at | |
1009 | * the same time. | |
1010 | */ | |
1011 | return 0; | |
f21fb3ed | 1012 | } |