]>
Commit | Line | Data |
---|---|---|
63697980 CM |
1 | /* |
2 | * Copyright 2012 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | */ | |
14 | ||
15 | #ifndef _GXIO_DMA_QUEUE_H_ | |
16 | #define _GXIO_DMA_QUEUE_H_ | |
17 | ||
18 | /* | |
19 | * DMA queue management APIs shared between TRIO and mPIPE. | |
20 | */ | |
21 | ||
a1ce3928 | 22 | #include <gxio/common.h> |
63697980 CM |
23 | |
24 | /* The credit counter lives in the high 32 bits. */ | |
25 | #define DMA_QUEUE_CREDIT_SHIFT 32 | |
26 | ||
27 | /* | |
28 | * State object that tracks a DMA queue's head and tail indices, as | |
29 | * well as the number of commands posted and completed. The | |
30 | * structure is accessed via a thread-safe, lock-free algorithm. | |
31 | */ | |
32 | typedef struct { | |
33 | /* | |
34 | * Address of a MPIPE_EDMA_POST_REGION_VAL_t, | |
35 | * TRIO_PUSH_DMA_REGION_VAL_t, or TRIO_PULL_DMA_REGION_VAL_t | |
36 | * register. These register have identical encodings and provide | |
37 | * information about how many commands have been processed. | |
38 | */ | |
39 | void *post_region_addr; | |
40 | ||
41 | /* | |
42 | * A lazily-updated count of how many edescs the hardware has | |
43 | * completed. | |
44 | */ | |
45 | uint64_t hw_complete_count __attribute__ ((aligned(64))); | |
46 | ||
47 | /* | |
48 | * High 32 bits are a count of available egress command credits, | |
49 | * low 24 bits are the next egress "slot". | |
50 | */ | |
51 | int64_t credits_and_next_index; | |
52 | ||
53 | } __gxio_dma_queue_t; | |
54 | ||
55 | /* Initialize a dma queue. */ | |
56 | extern void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue, | |
57 | void *post_region_addr, | |
58 | unsigned int num_entries); | |
59 | ||
60 | /* | |
61 | * Update the "credits_and_next_index" and "hw_complete_count" fields | |
62 | * based on pending hardware completions. Note that some other thread | |
63 | * may have already done this and, importantly, may still be in the | |
64 | * process of updating "credits_and_next_index". | |
65 | */ | |
66 | extern void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue); | |
67 | ||
68 | /* Wait for credits to become available. */ | |
69 | extern int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue, | |
70 | int64_t modifier); | |
71 | ||
72 | /* Reserve slots in the queue, optionally waiting for slots to become | |
73 | * available, and optionally returning a "completion_slot" suitable for | |
74 | * direct comparison to "hw_complete_count". | |
75 | */ | |
76 | static inline int64_t __gxio_dma_queue_reserve(__gxio_dma_queue_t *dma_queue, | |
77 | unsigned int num, bool wait, | |
78 | bool completion) | |
79 | { | |
80 | uint64_t slot; | |
81 | ||
82 | /* | |
83 | * Try to reserve 'num' egress command slots. We do this by | |
84 | * constructing a constant that subtracts N credits and adds N to | |
85 | * the index, and using fetchaddgez to only apply it if the credits | |
86 | * count doesn't go negative. | |
87 | */ | |
88 | int64_t modifier = (((int64_t)(-num)) << DMA_QUEUE_CREDIT_SHIFT) | num; | |
89 | int64_t old = | |
90 | __insn_fetchaddgez(&dma_queue->credits_and_next_index, | |
91 | modifier); | |
92 | ||
93 | if (unlikely(old + modifier < 0)) { | |
94 | /* | |
95 | * We're out of credits. Try once to get more by checking for | |
96 | * completed egress commands. If that fails, wait or fail. | |
97 | */ | |
98 | __gxio_dma_queue_update_credits(dma_queue); | |
99 | old = __insn_fetchaddgez(&dma_queue->credits_and_next_index, | |
100 | modifier); | |
101 | if (old + modifier < 0) { | |
102 | if (wait) | |
103 | old = __gxio_dma_queue_wait_for_credits | |
104 | (dma_queue, modifier); | |
105 | else | |
106 | return GXIO_ERR_DMA_CREDITS; | |
107 | } | |
108 | } | |
109 | ||
110 | /* The bottom 24 bits of old encode the "slot". */ | |
111 | slot = (old & 0xffffff); | |
112 | ||
113 | if (completion) { | |
114 | /* | |
115 | * A "completion_slot" is a "slot" which can be compared to | |
116 | * "hw_complete_count" at any time in the future. To convert | |
117 | * "slot" into a "completion_slot", we access "hw_complete_count" | |
118 | * once (knowing that we have reserved a slot, and thus, it will | |
119 | * be "basically" accurate), and combine its high 40 bits with | |
120 | * the 24 bit "slot", and handle "wrapping" by adding "1 << 24" | |
121 | * if the result is LESS than "hw_complete_count". | |
122 | */ | |
123 | uint64_t complete; | |
6aa7de05 | 124 | complete = READ_ONCE(dma_queue->hw_complete_count); |
63697980 CM |
125 | slot |= (complete & 0xffffffffff000000); |
126 | if (slot < complete) | |
127 | slot += 0x1000000; | |
128 | } | |
129 | ||
130 | /* | |
131 | * If any of our slots mod 256 were equivalent to 0, go ahead and | |
132 | * collect some egress credits, and update "hw_complete_count", and | |
133 | * make sure the index doesn't overflow into the credits. | |
134 | */ | |
135 | if (unlikely(((old + num) & 0xff) < num)) { | |
136 | __gxio_dma_queue_update_credits(dma_queue); | |
137 | ||
138 | /* Make sure the index doesn't overflow into the credits. */ | |
139 | #ifdef __BIG_ENDIAN__ | |
140 | *(((uint8_t *)&dma_queue->credits_and_next_index) + 4) = 0; | |
141 | #else | |
142 | *(((uint8_t *)&dma_queue->credits_and_next_index) + 3) = 0; | |
143 | #endif | |
144 | } | |
145 | ||
146 | return slot; | |
147 | } | |
148 | ||
149 | /* Non-inlinable "__gxio_dma_queue_reserve(..., true)". */ | |
150 | extern int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue, | |
151 | unsigned int num, int wait); | |
152 | ||
153 | /* Check whether a particular "completion slot" has completed. | |
154 | * | |
155 | * Note that this function requires a "completion slot", and thus | |
156 | * cannot be used with the result of any "reserve_fast" function. | |
157 | */ | |
158 | extern int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue, | |
159 | int64_t completion_slot, int update); | |
160 | ||
161 | #endif /* !_GXIO_DMA_QUEUE_H_ */ |