]>
Commit | Line | Data |
---|---|---|
b15a9f37 AK |
1 | /* |
2 | * Huawei HiNIC PCI Express Linux driver | |
3 | * Copyright(c) 2017 Huawei Technologies Co., Ltd | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
12 | * for more details. | |
13 | * | |
14 | */ | |
15 | ||
16 | #include <linux/kernel.h> | |
17 | #include <linux/types.h> | |
18 | #include <linux/pci.h> | |
19 | #include <linux/device.h> | |
20 | #include <linux/dma-mapping.h> | |
21 | #include <linux/slab.h> | |
22 | #include <linux/atomic.h> | |
23 | #include <linux/semaphore.h> | |
24 | #include <linux/errno.h> | |
25 | #include <linux/vmalloc.h> | |
76baca2e | 26 | #include <linux/err.h> |
b15a9f37 AK |
27 | #include <asm/byteorder.h> |
28 | ||
29 | #include "hinic_hw_if.h" | |
76baca2e | 30 | #include "hinic_hw_wqe.h" |
b15a9f37 | 31 | #include "hinic_hw_wq.h" |
d0b9805e | 32 | #include "hinic_hw_cmdq.h" |
b15a9f37 AK |
33 | |
34 | #define WQS_BLOCKS_PER_PAGE 4 | |
35 | ||
36 | #define WQ_BLOCK_SIZE 4096 | |
37 | #define WQS_PAGE_SIZE (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE) | |
38 | ||
39 | #define WQS_MAX_NUM_BLOCKS 128 | |
40 | #define WQS_FREE_BLOCKS_SIZE(wqs) (WQS_MAX_NUM_BLOCKS * \ | |
41 | sizeof((wqs)->free_blocks[0])) | |
42 | ||
43 | #define WQ_SIZE(wq) ((wq)->q_depth * (wq)->wqebb_size) | |
44 | ||
45 | #define WQ_PAGE_ADDR_SIZE sizeof(u64) | |
46 | #define WQ_MAX_PAGES (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) | |
47 | ||
d0b9805e AK |
48 | #define CMDQ_BLOCK_SIZE 512 |
49 | #define CMDQ_PAGE_SIZE 4096 | |
50 | ||
51 | #define CMDQ_WQ_MAX_PAGES (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) | |
52 | ||
b15a9f37 AK |
53 | #define WQ_BASE_VADDR(wqs, wq) \ |
54 | ((void *)((wqs)->page_vaddr[(wq)->page_idx]) \ | |
55 | + (wq)->block_idx * WQ_BLOCK_SIZE) | |
56 | ||
57 | #define WQ_BASE_PADDR(wqs, wq) \ | |
58 | ((wqs)->page_paddr[(wq)->page_idx] \ | |
59 | + (wq)->block_idx * WQ_BLOCK_SIZE) | |
60 | ||
61 | #define WQ_BASE_ADDR(wqs, wq) \ | |
62 | ((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \ | |
63 | + (wq)->block_idx * WQ_BLOCK_SIZE) | |
64 | ||
d0b9805e AK |
65 | #define CMDQ_BASE_VADDR(cmdq_pages, wq) \ |
66 | ((void *)((cmdq_pages)->page_vaddr) \ | |
67 | + (wq)->block_idx * CMDQ_BLOCK_SIZE) | |
68 | ||
69 | #define CMDQ_BASE_PADDR(cmdq_pages, wq) \ | |
70 | ((cmdq_pages)->page_paddr \ | |
71 | + (wq)->block_idx * CMDQ_BLOCK_SIZE) | |
72 | ||
73 | #define CMDQ_BASE_ADDR(cmdq_pages, wq) \ | |
74 | ((void *)((cmdq_pages)->shadow_page_vaddr) \ | |
75 | + (wq)->block_idx * CMDQ_BLOCK_SIZE) | |
76 | ||
76baca2e AK |
77 | #define WQ_PAGE_ADDR(wq, idx) \ |
78 | ((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)]) | |
79 | ||
80 | #define MASKED_WQE_IDX(wq, idx) ((idx) & (wq)->mask) | |
81 | ||
82 | #define WQE_IN_RANGE(wqe, start, end) \ | |
83 | (((unsigned long)(wqe) >= (unsigned long)(start)) && \ | |
84 | ((unsigned long)(wqe) < (unsigned long)(end))) | |
85 | ||
86 | #define WQE_SHADOW_PAGE(wq, wqe) \ | |
87 | (((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \ | |
88 | / (wq)->max_wqe_size) | |
89 | ||
ebda9b46 XC |
90 | static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx) |
91 | { | |
92 | return (((idx) & ((wq)->num_wqebbs_per_page - 1)) | |
93 | << (wq)->wqebb_size_shift); | |
94 | } | |
95 | ||
96 | static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx) | |
97 | { | |
98 | return (((idx) >> ((wq)->wqebbs_per_page_shift)) | |
99 | & ((wq)->num_q_pages - 1)); | |
100 | } | |
b15a9f37 AK |
101 | /** |
102 | * queue_alloc_page - allocate page for Queue | |
103 | * @hwif: HW interface for allocating DMA | |
104 | * @vaddr: virtual address will be returned in this address | |
105 | * @paddr: physical address will be returned in this address | |
106 | * @shadow_vaddr: VM area will be return here for holding WQ page addresses | |
107 | * @page_sz: page size of each WQ page | |
108 | * | |
109 | * Return 0 - Success, negative - Failure | |
110 | **/ | |
111 | static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr, | |
112 | void ***shadow_vaddr, size_t page_sz) | |
113 | { | |
114 | struct pci_dev *pdev = hwif->pdev; | |
115 | dma_addr_t dma_addr; | |
116 | ||
750afb08 LC |
117 | *vaddr = dma_alloc_coherent(&pdev->dev, page_sz, &dma_addr, |
118 | GFP_KERNEL); | |
b15a9f37 AK |
119 | if (!*vaddr) { |
120 | dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n"); | |
121 | return -ENOMEM; | |
122 | } | |
123 | ||
124 | *paddr = (u64)dma_addr; | |
125 | ||
126 | /* use vzalloc for big mem */ | |
127 | *shadow_vaddr = vzalloc(page_sz); | |
128 | if (!*shadow_vaddr) | |
129 | goto err_shadow_vaddr; | |
130 | ||
131 | return 0; | |
132 | ||
133 | err_shadow_vaddr: | |
134 | dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr); | |
135 | return -ENOMEM; | |
136 | } | |
137 | ||
138 | /** | |
139 | * wqs_allocate_page - allocate page for WQ set | |
140 | * @wqs: Work Queue Set | |
141 | * @page_idx: the page index of the page will be allocated | |
142 | * | |
143 | * Return 0 - Success, negative - Failure | |
144 | **/ | |
145 | static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx) | |
146 | { | |
147 | return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx], | |
148 | &wqs->page_paddr[page_idx], | |
149 | &wqs->shadow_page_vaddr[page_idx], | |
150 | WQS_PAGE_SIZE); | |
151 | } | |
152 | ||
153 | /** | |
154 | * wqs_free_page - free page of WQ set | |
155 | * @wqs: Work Queue Set | |
156 | * @page_idx: the page index of the page will be freed | |
157 | **/ | |
158 | static void wqs_free_page(struct hinic_wqs *wqs, int page_idx) | |
159 | { | |
160 | struct hinic_hwif *hwif = wqs->hwif; | |
161 | struct pci_dev *pdev = hwif->pdev; | |
162 | ||
163 | dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE, | |
164 | wqs->page_vaddr[page_idx], | |
165 | (dma_addr_t)wqs->page_paddr[page_idx]); | |
166 | vfree(wqs->shadow_page_vaddr[page_idx]); | |
167 | } | |
168 | ||
d0b9805e AK |
169 | /** |
170 | * cmdq_allocate_page - allocate page for cmdq | |
171 | * @cmdq_pages: the pages of the cmdq queue struct to hold the page | |
172 | * | |
173 | * Return 0 - Success, negative - Failure | |
174 | **/ | |
175 | static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages) | |
176 | { | |
177 | return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr, | |
178 | &cmdq_pages->page_paddr, | |
179 | &cmdq_pages->shadow_page_vaddr, | |
180 | CMDQ_PAGE_SIZE); | |
181 | } | |
182 | ||
183 | /** | |
184 | * cmdq_free_page - free page from cmdq | |
185 | * @cmdq_pages: the pages of the cmdq queue struct that hold the page | |
186 | * | |
187 | * Return 0 - Success, negative - Failure | |
188 | **/ | |
189 | static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages) | |
190 | { | |
191 | struct hinic_hwif *hwif = cmdq_pages->hwif; | |
192 | struct pci_dev *pdev = hwif->pdev; | |
193 | ||
194 | dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE, | |
195 | cmdq_pages->page_vaddr, | |
196 | (dma_addr_t)cmdq_pages->page_paddr); | |
197 | vfree(cmdq_pages->shadow_page_vaddr); | |
198 | } | |
199 | ||
b15a9f37 AK |
200 | static int alloc_page_arrays(struct hinic_wqs *wqs) |
201 | { | |
202 | struct hinic_hwif *hwif = wqs->hwif; | |
203 | struct pci_dev *pdev = hwif->pdev; | |
204 | size_t size; | |
205 | ||
206 | size = wqs->num_pages * sizeof(*wqs->page_paddr); | |
207 | wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); | |
208 | if (!wqs->page_paddr) | |
209 | return -ENOMEM; | |
210 | ||
211 | size = wqs->num_pages * sizeof(*wqs->page_vaddr); | |
212 | wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); | |
213 | if (!wqs->page_vaddr) | |
214 | goto err_page_vaddr; | |
215 | ||
216 | size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr); | |
217 | wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); | |
218 | if (!wqs->shadow_page_vaddr) | |
219 | goto err_page_shadow_vaddr; | |
220 | ||
221 | return 0; | |
222 | ||
223 | err_page_shadow_vaddr: | |
224 | devm_kfree(&pdev->dev, wqs->page_vaddr); | |
225 | ||
226 | err_page_vaddr: | |
227 | devm_kfree(&pdev->dev, wqs->page_paddr); | |
228 | return -ENOMEM; | |
229 | } | |
230 | ||
231 | static void free_page_arrays(struct hinic_wqs *wqs) | |
232 | { | |
233 | struct hinic_hwif *hwif = wqs->hwif; | |
234 | struct pci_dev *pdev = hwif->pdev; | |
235 | ||
236 | devm_kfree(&pdev->dev, wqs->shadow_page_vaddr); | |
237 | devm_kfree(&pdev->dev, wqs->page_vaddr); | |
238 | devm_kfree(&pdev->dev, wqs->page_paddr); | |
239 | } | |
240 | ||
241 | static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx, | |
242 | int *block_idx) | |
243 | { | |
244 | int pos; | |
245 | ||
246 | down(&wqs->alloc_blocks_lock); | |
247 | ||
248 | wqs->num_free_blks--; | |
249 | ||
250 | if (wqs->num_free_blks < 0) { | |
251 | wqs->num_free_blks++; | |
252 | up(&wqs->alloc_blocks_lock); | |
253 | return -ENOMEM; | |
254 | } | |
255 | ||
256 | pos = wqs->alloc_blk_pos++; | |
257 | pos &= WQS_MAX_NUM_BLOCKS - 1; | |
258 | ||
259 | *page_idx = wqs->free_blocks[pos].page_idx; | |
260 | *block_idx = wqs->free_blocks[pos].block_idx; | |
261 | ||
262 | wqs->free_blocks[pos].page_idx = -1; | |
263 | wqs->free_blocks[pos].block_idx = -1; | |
264 | ||
265 | up(&wqs->alloc_blocks_lock); | |
266 | return 0; | |
267 | } | |
268 | ||
269 | static void wqs_return_block(struct hinic_wqs *wqs, int page_idx, | |
270 | int block_idx) | |
271 | { | |
272 | int pos; | |
273 | ||
274 | down(&wqs->alloc_blocks_lock); | |
275 | ||
276 | pos = wqs->return_blk_pos++; | |
277 | pos &= WQS_MAX_NUM_BLOCKS - 1; | |
278 | ||
279 | wqs->free_blocks[pos].page_idx = page_idx; | |
280 | wqs->free_blocks[pos].block_idx = block_idx; | |
281 | ||
282 | wqs->num_free_blks++; | |
283 | ||
284 | up(&wqs->alloc_blocks_lock); | |
285 | } | |
286 | ||
287 | static void init_wqs_blocks_arr(struct hinic_wqs *wqs) | |
288 | { | |
289 | int page_idx, blk_idx, pos = 0; | |
290 | ||
291 | for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { | |
292 | for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) { | |
293 | wqs->free_blocks[pos].page_idx = page_idx; | |
294 | wqs->free_blocks[pos].block_idx = blk_idx; | |
295 | pos++; | |
296 | } | |
297 | } | |
298 | ||
299 | wqs->alloc_blk_pos = 0; | |
300 | wqs->return_blk_pos = pos; | |
301 | wqs->num_free_blks = pos; | |
302 | ||
303 | sema_init(&wqs->alloc_blocks_lock, 1); | |
304 | } | |
305 | ||
306 | /** | |
307 | * hinic_wqs_alloc - allocate Work Queues set | |
308 | * @wqs: Work Queue Set | |
309 | * @max_wqs: maximum wqs to allocate | |
310 | * @hwif: HW interface for use for the allocation | |
311 | * | |
312 | * Return 0 - Success, negative - Failure | |
313 | **/ | |
314 | int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs, | |
315 | struct hinic_hwif *hwif) | |
316 | { | |
317 | struct pci_dev *pdev = hwif->pdev; | |
318 | int err, i, page_idx; | |
319 | ||
320 | max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE); | |
321 | if (max_wqs > WQS_MAX_NUM_BLOCKS) { | |
322 | dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs); | |
323 | return -EINVAL; | |
324 | } | |
325 | ||
326 | wqs->hwif = hwif; | |
327 | wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE; | |
328 | ||
329 | if (alloc_page_arrays(wqs)) { | |
330 | dev_err(&pdev->dev, | |
331 | "Failed to allocate mem for page addresses\n"); | |
332 | return -ENOMEM; | |
333 | } | |
334 | ||
335 | for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { | |
336 | err = wqs_allocate_page(wqs, page_idx); | |
337 | if (err) { | |
338 | dev_err(&pdev->dev, "Failed wq page allocation\n"); | |
339 | goto err_wq_allocate_page; | |
340 | } | |
341 | } | |
342 | ||
343 | wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs), | |
344 | GFP_KERNEL); | |
345 | if (!wqs->free_blocks) { | |
346 | err = -ENOMEM; | |
347 | goto err_alloc_blocks; | |
348 | } | |
349 | ||
350 | init_wqs_blocks_arr(wqs); | |
351 | return 0; | |
352 | ||
353 | err_alloc_blocks: | |
354 | err_wq_allocate_page: | |
355 | for (i = 0; i < page_idx; i++) | |
356 | wqs_free_page(wqs, i); | |
357 | ||
358 | free_page_arrays(wqs); | |
359 | return err; | |
360 | } | |
361 | ||
362 | /** | |
363 | * hinic_wqs_free - free Work Queues set | |
364 | * @wqs: Work Queue Set | |
365 | **/ | |
366 | void hinic_wqs_free(struct hinic_wqs *wqs) | |
367 | { | |
368 | struct hinic_hwif *hwif = wqs->hwif; | |
369 | struct pci_dev *pdev = hwif->pdev; | |
370 | int page_idx; | |
371 | ||
372 | devm_kfree(&pdev->dev, wqs->free_blocks); | |
373 | ||
374 | for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) | |
375 | wqs_free_page(wqs, page_idx); | |
376 | ||
377 | free_page_arrays(wqs); | |
378 | } | |
379 | ||
380 | /** | |
381 | * alloc_wqes_shadow - allocate WQE shadows for WQ | |
382 | * @wq: WQ to allocate shadows for | |
383 | * | |
384 | * Return 0 - Success, negative - Failure | |
385 | **/ | |
386 | static int alloc_wqes_shadow(struct hinic_wq *wq) | |
387 | { | |
388 | struct hinic_hwif *hwif = wq->hwif; | |
389 | struct pci_dev *pdev = hwif->pdev; | |
390 | size_t size; | |
391 | ||
392 | size = wq->num_q_pages * wq->max_wqe_size; | |
393 | wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); | |
394 | if (!wq->shadow_wqe) | |
395 | return -ENOMEM; | |
396 | ||
397 | size = wq->num_q_pages * sizeof(wq->prod_idx); | |
398 | wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); | |
399 | if (!wq->shadow_idx) | |
400 | goto err_shadow_idx; | |
401 | ||
402 | return 0; | |
403 | ||
404 | err_shadow_idx: | |
405 | devm_kfree(&pdev->dev, wq->shadow_wqe); | |
406 | return -ENOMEM; | |
407 | } | |
408 | ||
409 | /** | |
410 | * free_wqes_shadow - free WQE shadows of WQ | |
411 | * @wq: WQ to free shadows from | |
412 | **/ | |
413 | static void free_wqes_shadow(struct hinic_wq *wq) | |
414 | { | |
415 | struct hinic_hwif *hwif = wq->hwif; | |
416 | struct pci_dev *pdev = hwif->pdev; | |
417 | ||
418 | devm_kfree(&pdev->dev, wq->shadow_idx); | |
419 | devm_kfree(&pdev->dev, wq->shadow_wqe); | |
420 | } | |
421 | ||
422 | /** | |
423 | * free_wq_pages - free pages of WQ | |
424 | * @hwif: HW interface for releasing dma addresses | |
425 | * @wq: WQ to free pages from | |
426 | * @num_q_pages: number pages to free | |
427 | **/ | |
428 | static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, | |
429 | int num_q_pages) | |
430 | { | |
431 | struct pci_dev *pdev = hwif->pdev; | |
432 | int i; | |
433 | ||
434 | for (i = 0; i < num_q_pages; i++) { | |
435 | void **vaddr = &wq->shadow_block_vaddr[i]; | |
436 | u64 *paddr = &wq->block_vaddr[i]; | |
437 | dma_addr_t dma_addr; | |
438 | ||
439 | dma_addr = (dma_addr_t)be64_to_cpu(*paddr); | |
440 | dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr, | |
441 | dma_addr); | |
442 | } | |
443 | ||
444 | free_wqes_shadow(wq); | |
445 | } | |
446 | ||
447 | /** | |
448 | * alloc_wq_pages - alloc pages for WQ | |
449 | * @hwif: HW interface for allocating dma addresses | |
450 | * @wq: WQ to allocate pages for | |
451 | * @max_pages: maximum pages allowed | |
452 | * | |
453 | * Return 0 - Success, negative - Failure | |
454 | **/ | |
455 | static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, | |
456 | int max_pages) | |
457 | { | |
458 | struct pci_dev *pdev = hwif->pdev; | |
459 | int i, err, num_q_pages; | |
460 | ||
461 | num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size; | |
462 | if (num_q_pages > max_pages) { | |
463 | dev_err(&pdev->dev, "Number wq pages exceeds the limit\n"); | |
464 | return -EINVAL; | |
465 | } | |
466 | ||
467 | if (num_q_pages & (num_q_pages - 1)) { | |
468 | dev_err(&pdev->dev, "Number wq pages must be power of 2\n"); | |
469 | return -EINVAL; | |
470 | } | |
471 | ||
472 | wq->num_q_pages = num_q_pages; | |
473 | ||
474 | err = alloc_wqes_shadow(wq); | |
475 | if (err) { | |
476 | dev_err(&pdev->dev, "Failed to allocate wqe shadow\n"); | |
477 | return err; | |
478 | } | |
479 | ||
480 | for (i = 0; i < num_q_pages; i++) { | |
481 | void **vaddr = &wq->shadow_block_vaddr[i]; | |
482 | u64 *paddr = &wq->block_vaddr[i]; | |
483 | dma_addr_t dma_addr; | |
484 | ||
750afb08 LC |
485 | *vaddr = dma_alloc_coherent(&pdev->dev, wq->wq_page_size, |
486 | &dma_addr, GFP_KERNEL); | |
b15a9f37 AK |
487 | if (!*vaddr) { |
488 | dev_err(&pdev->dev, "Failed to allocate wq page\n"); | |
489 | goto err_alloc_wq_pages; | |
490 | } | |
491 | ||
492 | /* HW uses Big Endian Format */ | |
493 | *paddr = cpu_to_be64(dma_addr); | |
494 | } | |
495 | ||
496 | return 0; | |
497 | ||
498 | err_alloc_wq_pages: | |
499 | free_wq_pages(wq, hwif, i); | |
500 | return -ENOMEM; | |
501 | } | |
502 | ||
503 | /** | |
504 | * hinic_wq_allocate - Allocate the WQ resources from the WQS | |
505 | * @wqs: WQ set from which to allocate the WQ resources | |
506 | * @wq: WQ to allocate resources for it from the WQ set | |
507 | * @wqebb_size: Work Queue Block Byte Size | |
508 | * @wq_page_size: the page size in the Work Queue | |
509 | * @q_depth: number of wqebbs in WQ | |
510 | * @max_wqe_size: maximum WQE size that will be used in the WQ | |
511 | * | |
512 | * Return 0 - Success, negative - Failure | |
513 | **/ | |
514 | int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq, | |
515 | u16 wqebb_size, u16 wq_page_size, u16 q_depth, | |
516 | u16 max_wqe_size) | |
517 | { | |
518 | struct hinic_hwif *hwif = wqs->hwif; | |
519 | struct pci_dev *pdev = hwif->pdev; | |
520 | u16 num_wqebbs_per_page; | |
ebda9b46 | 521 | u16 wqebb_size_shift; |
b15a9f37 AK |
522 | int err; |
523 | ||
ebda9b46 XC |
524 | if (!is_power_of_2(wqebb_size)) { |
525 | dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); | |
b15a9f37 AK |
526 | return -EINVAL; |
527 | } | |
528 | ||
529 | if (wq_page_size == 0) { | |
530 | dev_err(&pdev->dev, "wq_page_size must be > 0\n"); | |
531 | return -EINVAL; | |
532 | } | |
533 | ||
534 | if (q_depth & (q_depth - 1)) { | |
535 | dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); | |
536 | return -EINVAL; | |
537 | } | |
538 | ||
ebda9b46 XC |
539 | wqebb_size_shift = ilog2(wqebb_size); |
540 | num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) | |
541 | >> wqebb_size_shift; | |
b15a9f37 | 542 | |
ebda9b46 | 543 | if (!is_power_of_2(num_wqebbs_per_page)) { |
b15a9f37 AK |
544 | dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); |
545 | return -EINVAL; | |
546 | } | |
547 | ||
548 | wq->hwif = hwif; | |
549 | ||
550 | err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx); | |
551 | if (err) { | |
552 | dev_err(&pdev->dev, "Failed to get free wqs next block\n"); | |
553 | return err; | |
554 | } | |
555 | ||
556 | wq->wqebb_size = wqebb_size; | |
557 | wq->wq_page_size = wq_page_size; | |
558 | wq->q_depth = q_depth; | |
559 | wq->max_wqe_size = max_wqe_size; | |
560 | wq->num_wqebbs_per_page = num_wqebbs_per_page; | |
ebda9b46 XC |
561 | wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); |
562 | wq->wqebb_size_shift = wqebb_size_shift; | |
b15a9f37 AK |
563 | wq->block_vaddr = WQ_BASE_VADDR(wqs, wq); |
564 | wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq); | |
565 | wq->block_paddr = WQ_BASE_PADDR(wqs, wq); | |
566 | ||
567 | err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES); | |
568 | if (err) { | |
569 | dev_err(&pdev->dev, "Failed to allocate wq pages\n"); | |
570 | goto err_alloc_wq_pages; | |
571 | } | |
572 | ||
573 | atomic_set(&wq->cons_idx, 0); | |
574 | atomic_set(&wq->prod_idx, 0); | |
575 | atomic_set(&wq->delta, q_depth); | |
576 | wq->mask = q_depth - 1; | |
577 | ||
578 | return 0; | |
579 | ||
580 | err_alloc_wq_pages: | |
581 | wqs_return_block(wqs, wq->page_idx, wq->block_idx); | |
582 | return err; | |
583 | } | |
584 | ||
585 | /** | |
586 | * hinic_wq_free - Free the WQ resources to the WQS | |
587 | * @wqs: WQ set to free the WQ resources to it | |
588 | * @wq: WQ to free its resources to the WQ set resources | |
589 | **/ | |
590 | void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq) | |
591 | { | |
592 | free_wq_pages(wq, wqs->hwif, wq->num_q_pages); | |
593 | ||
594 | wqs_return_block(wqs, wq->page_idx, wq->block_idx); | |
595 | } | |
d0b9805e AK |
596 | |
597 | /** | |
598 | * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs | |
599 | * @cmdq_pages: will hold the pages of the cmdq | |
600 | * @wq: returned wqs | |
601 | * @hwif: HW interface | |
602 | * @cmdq_blocks: number of cmdq blocks/wq to allocate | |
603 | * @wqebb_size: Work Queue Block Byte Size | |
604 | * @wq_page_size: the page size in the Work Queue | |
605 | * @q_depth: number of wqebbs in WQ | |
606 | * @max_wqe_size: maximum WQE size that will be used in the WQ | |
607 | * | |
608 | * Return 0 - Success, negative - Failure | |
609 | **/ | |
610 | int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages, | |
611 | struct hinic_wq *wq, struct hinic_hwif *hwif, | |
612 | int cmdq_blocks, u16 wqebb_size, u16 wq_page_size, | |
613 | u16 q_depth, u16 max_wqe_size) | |
614 | { | |
615 | struct pci_dev *pdev = hwif->pdev; | |
ebda9b46 | 616 | u16 num_wqebbs_per_page_shift; |
d0b9805e | 617 | u16 num_wqebbs_per_page; |
ebda9b46 | 618 | u16 wqebb_size_shift; |
d0b9805e AK |
619 | int i, j, err = -ENOMEM; |
620 | ||
ebda9b46 XC |
621 | if (!is_power_of_2(wqebb_size)) { |
622 | dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); | |
d0b9805e AK |
623 | return -EINVAL; |
624 | } | |
625 | ||
626 | if (wq_page_size == 0) { | |
627 | dev_err(&pdev->dev, "wq_page_size must be > 0\n"); | |
628 | return -EINVAL; | |
629 | } | |
630 | ||
631 | if (q_depth & (q_depth - 1)) { | |
632 | dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); | |
633 | return -EINVAL; | |
634 | } | |
635 | ||
ebda9b46 XC |
636 | wqebb_size_shift = ilog2(wqebb_size); |
637 | num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) | |
638 | >> wqebb_size_shift; | |
d0b9805e | 639 | |
ebda9b46 | 640 | if (!is_power_of_2(num_wqebbs_per_page)) { |
d0b9805e AK |
641 | dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); |
642 | return -EINVAL; | |
643 | } | |
644 | ||
645 | cmdq_pages->hwif = hwif; | |
646 | ||
647 | err = cmdq_allocate_page(cmdq_pages); | |
648 | if (err) { | |
649 | dev_err(&pdev->dev, "Failed to allocate CMDQ page\n"); | |
650 | return err; | |
651 | } | |
ebda9b46 | 652 | num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); |
d0b9805e AK |
653 | |
654 | for (i = 0; i < cmdq_blocks; i++) { | |
655 | wq[i].hwif = hwif; | |
656 | wq[i].page_idx = 0; | |
657 | wq[i].block_idx = i; | |
658 | ||
659 | wq[i].wqebb_size = wqebb_size; | |
660 | wq[i].wq_page_size = wq_page_size; | |
661 | wq[i].q_depth = q_depth; | |
662 | wq[i].max_wqe_size = max_wqe_size; | |
663 | wq[i].num_wqebbs_per_page = num_wqebbs_per_page; | |
ebda9b46 XC |
664 | wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift; |
665 | wq[i].wqebb_size_shift = wqebb_size_shift; | |
d0b9805e AK |
666 | wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]); |
667 | wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]); | |
668 | wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]); | |
669 | ||
670 | err = alloc_wq_pages(&wq[i], cmdq_pages->hwif, | |
671 | CMDQ_WQ_MAX_PAGES); | |
672 | if (err) { | |
673 | dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n"); | |
674 | goto err_cmdq_block; | |
675 | } | |
676 | ||
677 | atomic_set(&wq[i].cons_idx, 0); | |
678 | atomic_set(&wq[i].prod_idx, 0); | |
679 | atomic_set(&wq[i].delta, q_depth); | |
680 | wq[i].mask = q_depth - 1; | |
681 | } | |
682 | ||
683 | return 0; | |
684 | ||
685 | err_cmdq_block: | |
686 | for (j = 0; j < i; j++) | |
687 | free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages); | |
688 | ||
689 | cmdq_free_page(cmdq_pages); | |
690 | return err; | |
691 | } | |
692 | ||
693 | /** | |
694 | * hinic_wqs_cmdq_free - Free wqs from cmdqs | |
695 | * @cmdq_pages: hold the pages of the cmdq | |
696 | * @wq: wqs to free | |
697 | * @cmdq_blocks: number of wqs to free | |
698 | **/ | |
699 | void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages, | |
700 | struct hinic_wq *wq, int cmdq_blocks) | |
701 | { | |
702 | int i; | |
703 | ||
704 | for (i = 0; i < cmdq_blocks; i++) | |
705 | free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages); | |
706 | ||
707 | cmdq_free_page(cmdq_pages); | |
708 | } | |
76baca2e AK |
709 | |
710 | static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr, | |
711 | int num_wqebbs, u16 idx) | |
712 | { | |
713 | void *wqebb_addr; | |
714 | int i; | |
715 | ||
716 | for (i = 0; i < num_wqebbs; i++, idx++) { | |
717 | idx = MASKED_WQE_IDX(wq, idx); | |
718 | wqebb_addr = WQ_PAGE_ADDR(wq, idx) + | |
719 | WQE_PAGE_OFF(wq, idx); | |
720 | ||
721 | memcpy(shadow_addr, wqebb_addr, wq->wqebb_size); | |
722 | ||
723 | shadow_addr += wq->wqebb_size; | |
724 | } | |
725 | } | |
726 | ||
727 | static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr, | |
728 | int num_wqebbs, u16 idx) | |
729 | { | |
730 | void *wqebb_addr; | |
731 | int i; | |
732 | ||
733 | for (i = 0; i < num_wqebbs; i++, idx++) { | |
734 | idx = MASKED_WQE_IDX(wq, idx); | |
735 | wqebb_addr = WQ_PAGE_ADDR(wq, idx) + | |
736 | WQE_PAGE_OFF(wq, idx); | |
737 | ||
738 | memcpy(wqebb_addr, shadow_addr, wq->wqebb_size); | |
739 | shadow_addr += wq->wqebb_size; | |
740 | } | |
741 | } | |
742 | ||
743 | /** | |
744 | * hinic_get_wqe - get wqe ptr in the current pi and update the pi | |
745 | * @wq: wq to get wqe from | |
746 | * @wqe_size: wqe size | |
747 | * @prod_idx: returned pi | |
748 | * | |
749 | * Return wqe pointer | |
750 | **/ | |
751 | struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size, | |
752 | u16 *prod_idx) | |
753 | { | |
754 | int curr_pg, end_pg, num_wqebbs; | |
755 | u16 curr_prod_idx, end_prod_idx; | |
756 | ||
757 | *prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx)); | |
758 | ||
ebda9b46 | 759 | num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift; |
76baca2e AK |
760 | |
761 | if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) { | |
762 | atomic_add(num_wqebbs, &wq->delta); | |
763 | return ERR_PTR(-EBUSY); | |
764 | } | |
765 | ||
766 | end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx); | |
767 | ||
768 | end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx); | |
769 | curr_prod_idx = end_prod_idx - num_wqebbs; | |
770 | curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx); | |
771 | ||
772 | /* end prod index points to the next wqebb, therefore minus 1 */ | |
773 | end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1); | |
774 | ||
775 | curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx); | |
776 | end_pg = WQE_PAGE_NUM(wq, end_prod_idx); | |
777 | ||
778 | *prod_idx = curr_prod_idx; | |
779 | ||
780 | if (curr_pg != end_pg) { | |
781 | void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; | |
782 | ||
783 | copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx); | |
784 | ||
785 | wq->shadow_idx[curr_pg] = *prod_idx; | |
786 | return shadow_addr; | |
787 | } | |
788 | ||
789 | return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx); | |
790 | } | |
791 | ||
cc18a754 ZC |
792 | /** |
793 | * hinic_return_wqe - return the wqe when transmit failed | |
794 | * @wq: wq to return wqe | |
795 | * @wqe_size: wqe size | |
796 | **/ | |
797 | void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size) | |
798 | { | |
799 | int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; | |
800 | ||
801 | atomic_sub(num_wqebbs, &wq->prod_idx); | |
802 | ||
803 | atomic_add(num_wqebbs, &wq->delta); | |
804 | } | |
805 | ||
76baca2e AK |
806 | /** |
807 | * hinic_put_wqe - return the wqe place to use for a new wqe | |
808 | * @wq: wq to return wqe | |
809 | * @wqe_size: wqe size | |
810 | **/ | |
811 | void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size) | |
812 | { | |
ebda9b46 XC |
813 | int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) |
814 | >> wq->wqebb_size_shift; | |
76baca2e AK |
815 | |
816 | atomic_add(num_wqebbs, &wq->cons_idx); | |
817 | ||
818 | atomic_add(num_wqebbs, &wq->delta); | |
819 | } | |
820 | ||
821 | /** | |
822 | * hinic_read_wqe - read wqe ptr in the current ci | |
823 | * @wq: wq to get read from | |
824 | * @wqe_size: wqe size | |
825 | * @cons_idx: returned ci | |
826 | * | |
827 | * Return wqe pointer | |
828 | **/ | |
829 | struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size, | |
830 | u16 *cons_idx) | |
831 | { | |
ebda9b46 XC |
832 | int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) |
833 | >> wq->wqebb_size_shift; | |
76baca2e AK |
834 | u16 curr_cons_idx, end_cons_idx; |
835 | int curr_pg, end_pg; | |
836 | ||
837 | if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth) | |
838 | return ERR_PTR(-EBUSY); | |
839 | ||
840 | curr_cons_idx = atomic_read(&wq->cons_idx); | |
841 | ||
842 | curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx); | |
843 | end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1); | |
844 | ||
845 | curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx); | |
846 | end_pg = WQE_PAGE_NUM(wq, end_cons_idx); | |
847 | ||
848 | *cons_idx = curr_cons_idx; | |
849 | ||
850 | if (curr_pg != end_pg) { | |
851 | void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; | |
852 | ||
853 | copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx); | |
854 | return shadow_addr; | |
855 | } | |
856 | ||
857 | return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx); | |
858 | } | |
859 | ||
e2585ea7 AK |
860 | /** |
861 | * hinic_read_wqe_direct - read wqe directly from ci position | |
862 | * @wq: wq | |
863 | * @cons_idx: ci position | |
864 | * | |
865 | * Return wqe | |
866 | **/ | |
867 | struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx) | |
868 | { | |
869 | return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx); | |
870 | } | |
871 | ||
76baca2e AK |
872 | /** |
873 | * wqe_shadow - check if a wqe is shadow | |
874 | * @wq: wq of the wqe | |
875 | * @wqe: the wqe for shadow checking | |
876 | * | |
877 | * Return true - shadow, false - Not shadow | |
878 | **/ | |
879 | static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe) | |
880 | { | |
881 | size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size; | |
882 | ||
883 | return WQE_IN_RANGE(wqe, wq->shadow_wqe, | |
884 | &wq->shadow_wqe[wqe_shadow_size]); | |
885 | } | |
886 | ||
887 | /** | |
888 | * hinic_write_wqe - write the wqe to the wq | |
889 | * @wq: wq to write wqe to | |
890 | * @wqe: wqe to write | |
891 | * @wqe_size: wqe size | |
892 | **/ | |
893 | void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe, | |
894 | unsigned int wqe_size) | |
895 | { | |
896 | int curr_pg, num_wqebbs; | |
897 | void *shadow_addr; | |
898 | u16 prod_idx; | |
899 | ||
900 | if (wqe_shadow(wq, wqe)) { | |
901 | curr_pg = WQE_SHADOW_PAGE(wq, wqe); | |
902 | ||
903 | prod_idx = wq->shadow_idx[curr_pg]; | |
904 | num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; | |
905 | shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; | |
906 | ||
907 | copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx); | |
908 | } | |
909 | } |