]>
Commit | Line | Data |
---|---|---|
244ab90e AL |
1 | /* |
2 | * DMA helper functions | |
3 | * | |
9c211ad2 | 4 | * Copyright (c) 2009, 2020 Red Hat |
244ab90e AL |
5 | * |
6 | * This work is licensed under the terms of the GNU General Public License | |
7 | * (GNU GPL), version 2 or later. | |
8 | */ | |
9 | ||
10 | #ifndef DMA_H | |
11 | #define DMA_H | |
12 | ||
022c62cb | 13 | #include "exec/memory.h" |
df32fd1c | 14 | #include "exec/address-spaces.h" |
737e150e | 15 | #include "block/block.h" |
5e5a94b6 | 16 | #include "block/accounting.h" |
244ab90e | 17 | |
43cf8ae6 DG |
18 | typedef enum { |
19 | DMA_DIRECTION_TO_DEVICE = 0, | |
20 | DMA_DIRECTION_FROM_DEVICE = 1, | |
21 | } DMADirection; | |
22 | ||
e5332e63 DG |
23 | /* |
24 | * When an IOMMU is present, bus addresses become distinct from | |
25 | * CPU/memory physical addresses and may be a different size. Because | |
26 | * the IOVA size depends more on the bus than on the platform, we more | |
27 | * or less have to treat these as 64-bit always to cover all (or at | |
28 | * least most) cases. | |
29 | */ | |
30 | typedef uint64_t dma_addr_t; | |
31 | ||
32 | #define DMA_ADDR_BITS 64 | |
33 | #define DMA_ADDR_FMT "%" PRIx64 | |
34 | ||
026644cf PMD |
35 | typedef struct ScatterGatherEntry ScatterGatherEntry; |
36 | ||
37 | struct QEMUSGList { | |
38 | ScatterGatherEntry *sg; | |
39 | int nsg; | |
40 | int nalloc; | |
bfa30f39 | 41 | dma_addr_t size; |
026644cf PMD |
42 | DeviceState *dev; |
43 | AddressSpace *as; | |
44 | }; | |
45 | ||
df32fd1c | 46 | static inline void dma_barrier(AddressSpace *as, DMADirection dir) |
7a0bac4d BH |
47 | { |
48 | /* | |
49 | * This is called before DMA read and write operations | |
50 | * unless the _relaxed form is used and is responsible | |
51 | * for providing some sane ordering of accesses vs | |
52 | * concurrently running VCPUs. | |
53 | * | |
54 | * Users of map(), unmap() or lower level st/ld_* | |
55 | * operations are responsible for providing their own | |
56 | * ordering via barriers. | |
57 | * | |
58 | * This primitive implementation does a simple smp_mb() | |
59 | * before each operation which provides pretty much full | |
60 | * ordering. | |
61 | * | |
62 | * A smarter implementation can be devised if needed to | |
63 | * use lighter barriers based on the direction of the | |
64 | * transfer, the DMA context, etc... | |
65 | */ | |
77ac58dd | 66 | smp_mb(); |
7a0bac4d BH |
67 | } |
68 | ||
d86a77f8 DG |
69 | /* Checks that the given range of addresses is valid for DMA. This is |
70 | * useful for certain cases, but usually you should just use | |
71 | * dma_memory_{read,write}() and check for errors */ | |
df32fd1c | 72 | static inline bool dma_memory_valid(AddressSpace *as, |
e5332e63 | 73 | dma_addr_t addr, dma_addr_t len, |
7ccb391c | 74 | DMADirection dir, MemTxAttrs attrs) |
d86a77f8 | 75 | { |
df32fd1c | 76 | return address_space_access_valid(as, addr, len, |
fddffa42 | 77 | dir == DMA_DIRECTION_FROM_DEVICE, |
7ccb391c | 78 | attrs); |
d86a77f8 DG |
79 | } |
80 | ||
9989bcd3 PMD |
81 | static inline MemTxResult dma_memory_rw_relaxed(AddressSpace *as, |
82 | dma_addr_t addr, | |
83 | void *buf, dma_addr_t len, | |
4afd0f2f PMD |
84 | DMADirection dir, |
85 | MemTxAttrs attrs) | |
d86a77f8 | 86 | { |
4afd0f2f | 87 | return address_space_rw(as, addr, attrs, |
9989bcd3 | 88 | buf, len, dir == DMA_DIRECTION_FROM_DEVICE); |
d86a77f8 DG |
89 | } |
90 | ||
b1f51303 PMD |
91 | static inline MemTxResult dma_memory_read_relaxed(AddressSpace *as, |
92 | dma_addr_t addr, | |
93 | void *buf, dma_addr_t len) | |
7a0bac4d | 94 | { |
4afd0f2f PMD |
95 | return dma_memory_rw_relaxed(as, addr, buf, len, |
96 | DMA_DIRECTION_TO_DEVICE, | |
97 | MEMTXATTRS_UNSPECIFIED); | |
7a0bac4d BH |
98 | } |
99 | ||
77c71d1d PMD |
100 | static inline MemTxResult dma_memory_write_relaxed(AddressSpace *as, |
101 | dma_addr_t addr, | |
102 | const void *buf, | |
103 | dma_addr_t len) | |
7a0bac4d | 104 | { |
df32fd1c | 105 | return dma_memory_rw_relaxed(as, addr, (void *)buf, len, |
4afd0f2f PMD |
106 | DMA_DIRECTION_FROM_DEVICE, |
107 | MEMTXATTRS_UNSPECIFIED); | |
7a0bac4d BH |
108 | } |
109 | ||
9989bcd3 PMD |
110 | /** |
111 | * dma_memory_rw: Read from or write to an address space from DMA controller. | |
112 | * | |
113 | * Return a MemTxResult indicating whether the operation succeeded | |
114 | * or failed (eg unassigned memory, device rejected the transaction, | |
115 | * IOMMU fault). | |
116 | * | |
117 | * @as: #AddressSpace to be accessed | |
118 | * @addr: address within that address space | |
119 | * @buf: buffer with the data transferred | |
120 | * @len: the number of bytes to read or write | |
121 | * @dir: indicates the transfer direction | |
23faf569 | 122 | * @attrs: memory transaction attributes |
9989bcd3 PMD |
123 | */ |
124 | static inline MemTxResult dma_memory_rw(AddressSpace *as, dma_addr_t addr, | |
125 | void *buf, dma_addr_t len, | |
23faf569 | 126 | DMADirection dir, MemTxAttrs attrs) |
7a0bac4d | 127 | { |
df32fd1c | 128 | dma_barrier(as, dir); |
7a0bac4d | 129 | |
23faf569 | 130 | return dma_memory_rw_relaxed(as, addr, buf, len, dir, attrs); |
7a0bac4d BH |
131 | } |
132 | ||
b1f51303 PMD |
133 | /** |
134 | * dma_memory_read: Read from an address space from DMA controller. | |
135 | * | |
136 | * Return a MemTxResult indicating whether the operation succeeded | |
137 | * or failed (eg unassigned memory, device rejected the transaction, | |
138 | * IOMMU fault). Called within RCU critical section. | |
139 | * | |
140 | * @as: #AddressSpace to be accessed | |
141 | * @addr: address within that address space | |
142 | * @buf: buffer with the data transferred | |
143 | * @len: length of the data transferred | |
ba06fe8a | 144 | * @attrs: memory transaction attributes |
b1f51303 PMD |
145 | */ |
146 | static inline MemTxResult dma_memory_read(AddressSpace *as, dma_addr_t addr, | |
ba06fe8a PMD |
147 | void *buf, dma_addr_t len, |
148 | MemTxAttrs attrs) | |
d86a77f8 | 149 | { |
23faf569 | 150 | return dma_memory_rw(as, addr, buf, len, |
ba06fe8a | 151 | DMA_DIRECTION_TO_DEVICE, attrs); |
d86a77f8 DG |
152 | } |
153 | ||
77c71d1d PMD |
154 | /** |
155 | * address_space_write: Write to address space from DMA controller. | |
156 | * | |
157 | * Return a MemTxResult indicating whether the operation succeeded | |
158 | * or failed (eg unassigned memory, device rejected the transaction, | |
159 | * IOMMU fault). | |
160 | * | |
161 | * @as: #AddressSpace to be accessed | |
162 | * @addr: address within that address space | |
163 | * @buf: buffer with the data transferred | |
164 | * @len: the number of bytes to write | |
ba06fe8a | 165 | * @attrs: memory transaction attributes |
77c71d1d PMD |
166 | */ |
167 | static inline MemTxResult dma_memory_write(AddressSpace *as, dma_addr_t addr, | |
ba06fe8a PMD |
168 | const void *buf, dma_addr_t len, |
169 | MemTxAttrs attrs) | |
d86a77f8 | 170 | { |
df32fd1c | 171 | return dma_memory_rw(as, addr, (void *)buf, len, |
ba06fe8a | 172 | DMA_DIRECTION_FROM_DEVICE, attrs); |
d86a77f8 DG |
173 | } |
174 | ||
bb755f52 PMD |
175 | /** |
176 | * dma_memory_set: Fill memory with a constant byte from DMA controller. | |
177 | * | |
178 | * Return a MemTxResult indicating whether the operation succeeded | |
179 | * or failed (eg unassigned memory, device rejected the transaction, | |
180 | * IOMMU fault). | |
181 | * | |
182 | * @as: #AddressSpace to be accessed | |
183 | * @addr: address within that address space | |
184 | * @c: constant byte to fill the memory | |
185 | * @len: the number of bytes to fill with the constant byte | |
7a36e42d | 186 | * @attrs: memory transaction attributes |
bb755f52 PMD |
187 | */ |
188 | MemTxResult dma_memory_set(AddressSpace *as, dma_addr_t addr, | |
7a36e42d | 189 | uint8_t c, dma_addr_t len, MemTxAttrs attrs); |
d86a77f8 | 190 | |
9c211ad2 PMD |
191 | /** |
192 | * address_space_map: Map a physical memory region into a host virtual address. | |
193 | * | |
194 | * May map a subset of the requested range, given by and returned in @plen. | |
195 | * May return %NULL and set *@plen to zero(0), if resources needed to perform | |
196 | * the mapping are exhausted. | |
197 | * Use only for reads OR writes - not for read-modify-write operations. | |
198 | * | |
199 | * @as: #AddressSpace to be accessed | |
200 | * @addr: address within that address space | |
201 | * @len: pointer to length of buffer; updated on return | |
202 | * @dir: indicates the transfer direction | |
a1d4b0a3 | 203 | * @attrs: memory attributes |
9c211ad2 | 204 | */ |
df32fd1c | 205 | static inline void *dma_memory_map(AddressSpace *as, |
d86a77f8 | 206 | dma_addr_t addr, dma_addr_t *len, |
a1d4b0a3 | 207 | DMADirection dir, MemTxAttrs attrs) |
d86a77f8 | 208 | { |
24addbc7 PB |
209 | hwaddr xlen = *len; |
210 | void *p; | |
211 | ||
f26404fb | 212 | p = address_space_map(as, addr, &xlen, dir == DMA_DIRECTION_FROM_DEVICE, |
a1d4b0a3 | 213 | attrs); |
24addbc7 PB |
214 | *len = xlen; |
215 | return p; | |
d86a77f8 DG |
216 | } |
217 | ||
9c211ad2 PMD |
218 | /** |
219 | * address_space_unmap: Unmaps a memory region previously mapped | |
220 | * by dma_memory_map() | |
221 | * | |
222 | * Will also mark the memory as dirty if @dir == %DMA_DIRECTION_FROM_DEVICE. | |
223 | * @access_len gives the amount of memory that was actually read or written | |
224 | * by the caller. | |
225 | * | |
226 | * @as: #AddressSpace used | |
227 | * @buffer: host pointer as returned by address_space_map() | |
228 | * @len: buffer length as returned by address_space_map() | |
229 | * @dir: indicates the transfer direction | |
230 | * @access_len: amount of data actually transferred | |
231 | */ | |
df32fd1c | 232 | static inline void dma_memory_unmap(AddressSpace *as, |
d86a77f8 DG |
233 | void *buffer, dma_addr_t len, |
234 | DMADirection dir, dma_addr_t access_len) | |
235 | { | |
df32fd1c | 236 | address_space_unmap(as, buffer, (hwaddr)len, |
24addbc7 | 237 | dir == DMA_DIRECTION_FROM_DEVICE, access_len); |
d86a77f8 DG |
238 | } |
239 | ||
240 | #define DEFINE_LDST_DMA(_lname, _sname, _bits, _end) \ | |
cd1db8df PMD |
241 | static inline MemTxResult ld##_lname##_##_end##_dma(AddressSpace *as, \ |
242 | dma_addr_t addr, \ | |
243 | uint##_bits##_t *pval, \ | |
244 | MemTxAttrs attrs) \ | |
245 | { \ | |
246 | MemTxResult res = dma_memory_read(as, addr, pval, (_bits) / 8, attrs); \ | |
247 | _end##_bits##_to_cpus(pval); \ | |
248 | return res; \ | |
249 | } \ | |
24aed6bc PMD |
250 | static inline MemTxResult st##_sname##_##_end##_dma(AddressSpace *as, \ |
251 | dma_addr_t addr, \ | |
252 | uint##_bits##_t val, \ | |
253 | MemTxAttrs attrs) \ | |
254 | { \ | |
255 | val = cpu_to_##_end##_bits(val); \ | |
256 | return dma_memory_write(as, addr, &val, (_bits) / 8, attrs); \ | |
d86a77f8 DG |
257 | } |
258 | ||
cd1db8df PMD |
259 | static inline MemTxResult ldub_dma(AddressSpace *as, dma_addr_t addr, |
260 | uint8_t *val, MemTxAttrs attrs) | |
d86a77f8 | 261 | { |
cd1db8df | 262 | return dma_memory_read(as, addr, val, 1, attrs); |
d86a77f8 DG |
263 | } |
264 | ||
24aed6bc PMD |
265 | static inline MemTxResult stb_dma(AddressSpace *as, dma_addr_t addr, |
266 | uint8_t val, MemTxAttrs attrs) | |
d86a77f8 | 267 | { |
24aed6bc | 268 | return dma_memory_write(as, addr, &val, 1, attrs); |
d86a77f8 DG |
269 | } |
270 | ||
271 | DEFINE_LDST_DMA(uw, w, 16, le); | |
272 | DEFINE_LDST_DMA(l, l, 32, le); | |
273 | DEFINE_LDST_DMA(q, q, 64, le); | |
274 | DEFINE_LDST_DMA(uw, w, 16, be); | |
275 | DEFINE_LDST_DMA(l, l, 32, be); | |
276 | DEFINE_LDST_DMA(q, q, 64, be); | |
277 | ||
278 | #undef DEFINE_LDST_DMA | |
279 | ||
10dc8aef | 280 | struct ScatterGatherEntry { |
d3231181 DG |
281 | dma_addr_t base; |
282 | dma_addr_t len; | |
10dc8aef | 283 | }; |
244ab90e | 284 | |
f487b677 PB |
285 | void qemu_sglist_init(QEMUSGList *qsg, DeviceState *dev, int alloc_hint, |
286 | AddressSpace *as); | |
d3231181 | 287 | void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len); |
244ab90e AL |
288 | void qemu_sglist_destroy(QEMUSGList *qsg); |
289 | ||
8a8e63eb PB |
290 | typedef BlockAIOCB *DMAIOFunc(int64_t offset, QEMUIOVector *iov, |
291 | BlockCompletionFunc *cb, void *cb_opaque, | |
292 | void *opaque); | |
7c84b1b8 | 293 | |
8a8e63eb | 294 | BlockAIOCB *dma_blk_io(AioContext *ctx, |
99868af3 | 295 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
8a8e63eb PB |
296 | DMAIOFunc *io_func, void *io_func_opaque, |
297 | BlockCompletionFunc *cb, void *opaque, DMADirection dir); | |
4be74634 | 298 | BlockAIOCB *dma_blk_read(BlockBackend *blk, |
99868af3 | 299 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
4be74634 MA |
300 | BlockCompletionFunc *cb, void *opaque); |
301 | BlockAIOCB *dma_blk_write(BlockBackend *blk, | |
99868af3 | 302 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
097310b5 | 303 | BlockCompletionFunc *cb, void *opaque); |
f02b664a | 304 | MemTxResult dma_buf_read(void *ptr, dma_addr_t len, dma_addr_t *residual, |
bfa30f39 | 305 | QEMUSGList *sg, MemTxAttrs attrs); |
f02b664a | 306 | MemTxResult dma_buf_write(void *ptr, dma_addr_t len, dma_addr_t *residual, |
bfa30f39 | 307 | QEMUSGList *sg, MemTxAttrs attrs); |
8171ee35 | 308 | |
4be74634 | 309 | void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie, |
84a69356 PB |
310 | QEMUSGList *sg, enum BlockAcctType type); |
311 | ||
f14fb6c2 EA |
312 | /** |
313 | * dma_aligned_pow2_mask: Return the address bit mask of the largest | |
314 | * power of 2 size less or equal than @end - @start + 1, aligned with @start, | |
315 | * and bounded by 1 << @max_addr_bits bits. | |
316 | * | |
317 | * @start: range start address | |
318 | * @end: range end address (greater than @start) | |
319 | * @max_addr_bits: max address bits (<= 64) | |
320 | */ | |
321 | uint64_t dma_aligned_pow2_mask(uint64_t start, uint64_t end, | |
322 | int max_addr_bits); | |
323 | ||
244ab90e | 324 | #endif |