]>
Commit | Line | Data |
---|---|---|
bdd6a90a FZ |
1 | /* |
2 | * NVMe block driver based on vfio | |
3 | * | |
4 | * Copyright 2016 - 2018 Red Hat, Inc. | |
5 | * | |
6 | * Authors: | |
7 | * Fam Zheng <famz@redhat.com> | |
8 | * Paolo Bonzini <pbonzini@redhat.com> | |
9 | * | |
10 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
11 | * See the COPYING file in the top-level directory. | |
12 | */ | |
13 | ||
14 | #include "qemu/osdep.h" | |
15 | #include <linux/vfio.h> | |
16 | #include "qapi/error.h" | |
17 | #include "qapi/qmp/qdict.h" | |
18 | #include "qapi/qmp/qstring.h" | |
19 | #include "qemu/error-report.h" | |
db725815 | 20 | #include "qemu/main-loop.h" |
0b8fa32f | 21 | #include "qemu/module.h" |
bdd6a90a | 22 | #include "qemu/cutils.h" |
922a01a0 | 23 | #include "qemu/option.h" |
bdd6a90a FZ |
24 | #include "qemu/vfio-helpers.h" |
25 | #include "block/block_int.h" | |
e4ec5ad4 | 26 | #include "sysemu/replay.h" |
bdd6a90a FZ |
27 | #include "trace.h" |
28 | ||
a3d9a352 | 29 | #include "block/nvme.h" |
bdd6a90a FZ |
30 | |
31 | #define NVME_SQ_ENTRY_BYTES 64 | |
32 | #define NVME_CQ_ENTRY_BYTES 16 | |
33 | #define NVME_QUEUE_SIZE 128 | |
34 | #define NVME_BAR_SIZE 8192 | |
35 | ||
1086e95d SH |
36 | /* |
37 | * We have to leave one slot empty as that is the full queue case where | |
38 | * head == tail + 1. | |
39 | */ | |
40 | #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1) | |
41 | ||
b75fd5f5 SH |
42 | typedef struct BDRVNVMeState BDRVNVMeState; |
43 | ||
bdd6a90a FZ |
44 | typedef struct { |
45 | int32_t head, tail; | |
46 | uint8_t *queue; | |
47 | uint64_t iova; | |
48 | /* Hardware MMIO register */ | |
49 | volatile uint32_t *doorbell; | |
50 | } NVMeQueue; | |
51 | ||
52 | typedef struct { | |
53 | BlockCompletionFunc *cb; | |
54 | void *opaque; | |
55 | int cid; | |
56 | void *prp_list_page; | |
57 | uint64_t prp_list_iova; | |
1086e95d | 58 | int free_req_next; /* q->reqs[] index of next free req */ |
bdd6a90a FZ |
59 | } NVMeRequest; |
60 | ||
61 | typedef struct { | |
bdd6a90a FZ |
62 | QemuMutex lock; |
63 | ||
b75fd5f5 SH |
64 | /* Read from I/O code path, initialized under BQL */ |
65 | BDRVNVMeState *s; | |
66 | int index; | |
67 | ||
bdd6a90a | 68 | /* Fields protected by BQL */ |
bdd6a90a FZ |
69 | uint8_t *prp_list_pages; |
70 | ||
71 | /* Fields protected by @lock */ | |
a5db74f3 | 72 | CoQueue free_req_queue; |
bdd6a90a FZ |
73 | NVMeQueue sq, cq; |
74 | int cq_phase; | |
1086e95d SH |
75 | int free_req_head; |
76 | NVMeRequest reqs[NVME_NUM_REQS]; | |
bdd6a90a FZ |
77 | bool busy; |
78 | int need_kick; | |
79 | int inflight; | |
80 | } NVMeQueuePair; | |
81 | ||
82 | /* Memory mapped registers */ | |
83 | typedef volatile struct { | |
84 | uint64_t cap; | |
85 | uint32_t vs; | |
86 | uint32_t intms; | |
87 | uint32_t intmc; | |
88 | uint32_t cc; | |
89 | uint32_t reserved0; | |
90 | uint32_t csts; | |
91 | uint32_t nssr; | |
92 | uint32_t aqa; | |
93 | uint64_t asq; | |
94 | uint64_t acq; | |
95 | uint32_t cmbloc; | |
96 | uint32_t cmbsz; | |
97 | uint8_t reserved1[0xec0]; | |
98 | uint8_t cmd_set_specfic[0x100]; | |
99 | uint32_t doorbells[]; | |
83c68e14 | 100 | } NVMeRegs; |
bdd6a90a FZ |
101 | |
102 | QEMU_BUILD_BUG_ON(offsetof(NVMeRegs, doorbells) != 0x1000); | |
103 | ||
b75fd5f5 | 104 | struct BDRVNVMeState { |
bdd6a90a FZ |
105 | AioContext *aio_context; |
106 | QEMUVFIOState *vfio; | |
107 | NVMeRegs *regs; | |
108 | /* The submission/completion queue pairs. | |
109 | * [0]: admin queue. | |
110 | * [1..]: io queues. | |
111 | */ | |
112 | NVMeQueuePair **queues; | |
113 | int nr_queues; | |
114 | size_t page_size; | |
115 | /* How many uint32_t elements does each doorbell entry take. */ | |
116 | size_t doorbell_scale; | |
117 | bool write_cache_supported; | |
118 | EventNotifier irq_notifier; | |
118d1b6a | 119 | |
bdd6a90a FZ |
120 | uint64_t nsze; /* Namespace size reported by identify command */ |
121 | int nsid; /* The namespace id to read/write data. */ | |
1120407b | 122 | int blkshift; |
118d1b6a | 123 | |
bdd6a90a | 124 | uint64_t max_transfer; |
2f0d8947 | 125 | bool plugged; |
bdd6a90a | 126 | |
e0dd95e3 | 127 | bool supports_write_zeroes; |
e87a09d6 | 128 | bool supports_discard; |
e0dd95e3 | 129 | |
bdd6a90a FZ |
130 | CoMutex dma_map_lock; |
131 | CoQueue dma_flush_queue; | |
132 | ||
133 | /* Total size of mapped qiov, accessed under dma_map_lock */ | |
134 | int dma_map_count; | |
cc61b074 HR |
135 | |
136 | /* PCI address (required for nvme_refresh_filename()) */ | |
137 | char *device; | |
b75fd5f5 | 138 | }; |
bdd6a90a FZ |
139 | |
140 | #define NVME_BLOCK_OPT_DEVICE "device" | |
141 | #define NVME_BLOCK_OPT_NAMESPACE "namespace" | |
142 | ||
143 | static QemuOptsList runtime_opts = { | |
144 | .name = "nvme", | |
145 | .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head), | |
146 | .desc = { | |
147 | { | |
148 | .name = NVME_BLOCK_OPT_DEVICE, | |
149 | .type = QEMU_OPT_STRING, | |
150 | .help = "NVMe PCI device address", | |
151 | }, | |
152 | { | |
153 | .name = NVME_BLOCK_OPT_NAMESPACE, | |
154 | .type = QEMU_OPT_NUMBER, | |
155 | .help = "NVMe namespace", | |
156 | }, | |
157 | { /* end of list */ } | |
158 | }, | |
159 | }; | |
160 | ||
161 | static void nvme_init_queue(BlockDriverState *bs, NVMeQueue *q, | |
162 | int nentries, int entry_bytes, Error **errp) | |
163 | { | |
164 | BDRVNVMeState *s = bs->opaque; | |
165 | size_t bytes; | |
166 | int r; | |
167 | ||
168 | bytes = ROUND_UP(nentries * entry_bytes, s->page_size); | |
169 | q->head = q->tail = 0; | |
170 | q->queue = qemu_try_blockalign0(bs, bytes); | |
171 | ||
172 | if (!q->queue) { | |
173 | error_setg(errp, "Cannot allocate queue"); | |
174 | return; | |
175 | } | |
176 | r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova); | |
177 | if (r) { | |
178 | error_setg(errp, "Cannot map queue"); | |
179 | } | |
180 | } | |
181 | ||
b75fd5f5 | 182 | static void nvme_free_queue_pair(NVMeQueuePair *q) |
bdd6a90a FZ |
183 | { |
184 | qemu_vfree(q->prp_list_pages); | |
185 | qemu_vfree(q->sq.queue); | |
186 | qemu_vfree(q->cq.queue); | |
187 | qemu_mutex_destroy(&q->lock); | |
188 | g_free(q); | |
189 | } | |
190 | ||
191 | static void nvme_free_req_queue_cb(void *opaque) | |
192 | { | |
193 | NVMeQueuePair *q = opaque; | |
194 | ||
195 | qemu_mutex_lock(&q->lock); | |
196 | while (qemu_co_enter_next(&q->free_req_queue, &q->lock)) { | |
197 | /* Retry all pending requests */ | |
198 | } | |
199 | qemu_mutex_unlock(&q->lock); | |
200 | } | |
201 | ||
202 | static NVMeQueuePair *nvme_create_queue_pair(BlockDriverState *bs, | |
203 | int idx, int size, | |
204 | Error **errp) | |
205 | { | |
206 | int i, r; | |
207 | BDRVNVMeState *s = bs->opaque; | |
208 | Error *local_err = NULL; | |
209 | NVMeQueuePair *q = g_new0(NVMeQueuePair, 1); | |
210 | uint64_t prp_list_iova; | |
211 | ||
212 | qemu_mutex_init(&q->lock); | |
b75fd5f5 | 213 | q->s = s; |
bdd6a90a FZ |
214 | q->index = idx; |
215 | qemu_co_queue_init(&q->free_req_queue); | |
1086e95d | 216 | q->prp_list_pages = qemu_blockalign0(bs, s->page_size * NVME_NUM_REQS); |
bdd6a90a | 217 | r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages, |
1086e95d | 218 | s->page_size * NVME_NUM_REQS, |
bdd6a90a FZ |
219 | false, &prp_list_iova); |
220 | if (r) { | |
221 | goto fail; | |
222 | } | |
1086e95d SH |
223 | q->free_req_head = -1; |
224 | for (i = 0; i < NVME_NUM_REQS; i++) { | |
bdd6a90a FZ |
225 | NVMeRequest *req = &q->reqs[i]; |
226 | req->cid = i + 1; | |
1086e95d SH |
227 | req->free_req_next = q->free_req_head; |
228 | q->free_req_head = i; | |
bdd6a90a FZ |
229 | req->prp_list_page = q->prp_list_pages + i * s->page_size; |
230 | req->prp_list_iova = prp_list_iova + i * s->page_size; | |
231 | } | |
1086e95d | 232 | |
bdd6a90a FZ |
233 | nvme_init_queue(bs, &q->sq, size, NVME_SQ_ENTRY_BYTES, &local_err); |
234 | if (local_err) { | |
235 | error_propagate(errp, local_err); | |
236 | goto fail; | |
237 | } | |
238 | q->sq.doorbell = &s->regs->doorbells[idx * 2 * s->doorbell_scale]; | |
239 | ||
240 | nvme_init_queue(bs, &q->cq, size, NVME_CQ_ENTRY_BYTES, &local_err); | |
241 | if (local_err) { | |
242 | error_propagate(errp, local_err); | |
243 | goto fail; | |
244 | } | |
461bba04 | 245 | q->cq.doorbell = &s->regs->doorbells[(idx * 2 + 1) * s->doorbell_scale]; |
bdd6a90a FZ |
246 | |
247 | return q; | |
248 | fail: | |
b75fd5f5 | 249 | nvme_free_queue_pair(q); |
bdd6a90a FZ |
250 | return NULL; |
251 | } | |
252 | ||
253 | /* With q->lock */ | |
b75fd5f5 | 254 | static void nvme_kick(NVMeQueuePair *q) |
bdd6a90a | 255 | { |
b75fd5f5 SH |
256 | BDRVNVMeState *s = q->s; |
257 | ||
bdd6a90a FZ |
258 | if (s->plugged || !q->need_kick) { |
259 | return; | |
260 | } | |
261 | trace_nvme_kick(s, q->index); | |
262 | assert(!(q->sq.tail & 0xFF00)); | |
263 | /* Fence the write to submission queue entry before notifying the device. */ | |
264 | smp_wmb(); | |
265 | *q->sq.doorbell = cpu_to_le32(q->sq.tail); | |
266 | q->inflight += q->need_kick; | |
267 | q->need_kick = 0; | |
268 | } | |
269 | ||
270 | /* Find a free request element if any, otherwise: | |
271 | * a) if in coroutine context, try to wait for one to become available; | |
272 | * b) if not in coroutine, return NULL; | |
273 | */ | |
274 | static NVMeRequest *nvme_get_free_req(NVMeQueuePair *q) | |
275 | { | |
1086e95d | 276 | NVMeRequest *req; |
bdd6a90a FZ |
277 | |
278 | qemu_mutex_lock(&q->lock); | |
1086e95d SH |
279 | |
280 | while (q->free_req_head == -1) { | |
bdd6a90a FZ |
281 | if (qemu_in_coroutine()) { |
282 | trace_nvme_free_req_queue_wait(q); | |
283 | qemu_co_queue_wait(&q->free_req_queue, &q->lock); | |
284 | } else { | |
285 | qemu_mutex_unlock(&q->lock); | |
286 | return NULL; | |
287 | } | |
288 | } | |
1086e95d SH |
289 | |
290 | req = &q->reqs[q->free_req_head]; | |
291 | q->free_req_head = req->free_req_next; | |
292 | req->free_req_next = -1; | |
293 | ||
bdd6a90a FZ |
294 | qemu_mutex_unlock(&q->lock); |
295 | return req; | |
296 | } | |
297 | ||
1086e95d SH |
298 | /* With q->lock */ |
299 | static void nvme_put_free_req_locked(NVMeQueuePair *q, NVMeRequest *req) | |
300 | { | |
301 | req->free_req_next = q->free_req_head; | |
302 | q->free_req_head = req - q->reqs; | |
303 | } | |
304 | ||
305 | /* With q->lock */ | |
b75fd5f5 | 306 | static void nvme_wake_free_req_locked(NVMeQueuePair *q) |
1086e95d SH |
307 | { |
308 | if (!qemu_co_queue_empty(&q->free_req_queue)) { | |
b75fd5f5 | 309 | replay_bh_schedule_oneshot_event(q->s->aio_context, |
1086e95d SH |
310 | nvme_free_req_queue_cb, q); |
311 | } | |
312 | } | |
313 | ||
314 | /* Insert a request in the freelist and wake waiters */ | |
b75fd5f5 | 315 | static void nvme_put_free_req_and_wake(NVMeQueuePair *q, NVMeRequest *req) |
1086e95d SH |
316 | { |
317 | qemu_mutex_lock(&q->lock); | |
318 | nvme_put_free_req_locked(q, req); | |
b75fd5f5 | 319 | nvme_wake_free_req_locked(q); |
1086e95d SH |
320 | qemu_mutex_unlock(&q->lock); |
321 | } | |
322 | ||
bdd6a90a FZ |
323 | static inline int nvme_translate_error(const NvmeCqe *c) |
324 | { | |
325 | uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF; | |
326 | if (status) { | |
327 | trace_nvme_error(le32_to_cpu(c->result), | |
328 | le16_to_cpu(c->sq_head), | |
329 | le16_to_cpu(c->sq_id), | |
330 | le16_to_cpu(c->cid), | |
331 | le16_to_cpu(status)); | |
332 | } | |
333 | switch (status) { | |
334 | case 0: | |
335 | return 0; | |
336 | case 1: | |
337 | return -ENOSYS; | |
338 | case 2: | |
339 | return -EINVAL; | |
340 | default: | |
341 | return -EIO; | |
342 | } | |
343 | } | |
344 | ||
345 | /* With q->lock */ | |
b75fd5f5 | 346 | static bool nvme_process_completion(NVMeQueuePair *q) |
bdd6a90a | 347 | { |
b75fd5f5 | 348 | BDRVNVMeState *s = q->s; |
bdd6a90a FZ |
349 | bool progress = false; |
350 | NVMeRequest *preq; | |
351 | NVMeRequest req; | |
352 | NvmeCqe *c; | |
353 | ||
354 | trace_nvme_process_completion(s, q->index, q->inflight); | |
355 | if (q->busy || s->plugged) { | |
356 | trace_nvme_process_completion_queue_busy(s, q->index); | |
357 | return false; | |
358 | } | |
359 | q->busy = true; | |
360 | assert(q->inflight >= 0); | |
361 | while (q->inflight) { | |
04b3fb39 | 362 | int ret; |
bdd6a90a | 363 | int16_t cid; |
04b3fb39 | 364 | |
bdd6a90a | 365 | c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES]; |
258867d1 | 366 | if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) { |
bdd6a90a FZ |
367 | break; |
368 | } | |
04b3fb39 | 369 | ret = nvme_translate_error(c); |
bdd6a90a FZ |
370 | q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE; |
371 | if (!q->cq.head) { | |
372 | q->cq_phase = !q->cq_phase; | |
373 | } | |
374 | cid = le16_to_cpu(c->cid); | |
375 | if (cid == 0 || cid > NVME_QUEUE_SIZE) { | |
376 | fprintf(stderr, "Unexpected CID in completion queue: %" PRIu32 "\n", | |
377 | cid); | |
378 | continue; | |
379 | } | |
bdd6a90a FZ |
380 | trace_nvme_complete_command(s, q->index, cid); |
381 | preq = &q->reqs[cid - 1]; | |
382 | req = *preq; | |
383 | assert(req.cid == cid); | |
384 | assert(req.cb); | |
1086e95d | 385 | nvme_put_free_req_locked(q, preq); |
bdd6a90a FZ |
386 | preq->cb = preq->opaque = NULL; |
387 | qemu_mutex_unlock(&q->lock); | |
04b3fb39 | 388 | req.cb(req.opaque, ret); |
bdd6a90a | 389 | qemu_mutex_lock(&q->lock); |
bdd6a90a | 390 | q->inflight--; |
bdd6a90a FZ |
391 | progress = true; |
392 | } | |
393 | if (progress) { | |
394 | /* Notify the device so it can post more completions. */ | |
395 | smp_mb_release(); | |
396 | *q->cq.doorbell = cpu_to_le32(q->cq.head); | |
b75fd5f5 | 397 | nvme_wake_free_req_locked(q); |
bdd6a90a FZ |
398 | } |
399 | q->busy = false; | |
400 | return progress; | |
401 | } | |
402 | ||
403 | static void nvme_trace_command(const NvmeCmd *cmd) | |
404 | { | |
405 | int i; | |
406 | ||
407 | for (i = 0; i < 8; ++i) { | |
408 | uint8_t *cmdp = (uint8_t *)cmd + i * 8; | |
409 | trace_nvme_submit_command_raw(cmdp[0], cmdp[1], cmdp[2], cmdp[3], | |
410 | cmdp[4], cmdp[5], cmdp[6], cmdp[7]); | |
411 | } | |
412 | } | |
413 | ||
b75fd5f5 | 414 | static void nvme_submit_command(NVMeQueuePair *q, NVMeRequest *req, |
bdd6a90a FZ |
415 | NvmeCmd *cmd, BlockCompletionFunc cb, |
416 | void *opaque) | |
417 | { | |
418 | assert(!req->cb); | |
419 | req->cb = cb; | |
420 | req->opaque = opaque; | |
421 | cmd->cid = cpu_to_le32(req->cid); | |
422 | ||
b75fd5f5 | 423 | trace_nvme_submit_command(q->s, q->index, req->cid); |
bdd6a90a FZ |
424 | nvme_trace_command(cmd); |
425 | qemu_mutex_lock(&q->lock); | |
426 | memcpy((uint8_t *)q->sq.queue + | |
427 | q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd)); | |
428 | q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE; | |
429 | q->need_kick++; | |
b75fd5f5 SH |
430 | nvme_kick(q); |
431 | nvme_process_completion(q); | |
bdd6a90a FZ |
432 | qemu_mutex_unlock(&q->lock); |
433 | } | |
434 | ||
435 | static void nvme_cmd_sync_cb(void *opaque, int ret) | |
436 | { | |
437 | int *pret = opaque; | |
438 | *pret = ret; | |
4720cbee | 439 | aio_wait_kick(); |
bdd6a90a FZ |
440 | } |
441 | ||
442 | static int nvme_cmd_sync(BlockDriverState *bs, NVMeQueuePair *q, | |
443 | NvmeCmd *cmd) | |
444 | { | |
445 | NVMeRequest *req; | |
bdd6a90a FZ |
446 | int ret = -EINPROGRESS; |
447 | req = nvme_get_free_req(q); | |
448 | if (!req) { | |
449 | return -EBUSY; | |
450 | } | |
b75fd5f5 | 451 | nvme_submit_command(q, req, cmd, nvme_cmd_sync_cb, &ret); |
bdd6a90a FZ |
452 | |
453 | BDRV_POLL_WHILE(bs, ret == -EINPROGRESS); | |
454 | return ret; | |
455 | } | |
456 | ||
457 | static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp) | |
458 | { | |
459 | BDRVNVMeState *s = bs->opaque; | |
460 | NvmeIdCtrl *idctrl; | |
461 | NvmeIdNs *idns; | |
118d1b6a | 462 | NvmeLBAF *lbaf; |
bdd6a90a | 463 | uint8_t *resp; |
e0dd95e3 | 464 | uint16_t oncs; |
1120407b | 465 | int r; |
bdd6a90a FZ |
466 | uint64_t iova; |
467 | NvmeCmd cmd = { | |
468 | .opcode = NVME_ADM_CMD_IDENTIFY, | |
469 | .cdw10 = cpu_to_le32(0x1), | |
470 | }; | |
471 | ||
472 | resp = qemu_try_blockalign0(bs, sizeof(NvmeIdCtrl)); | |
473 | if (!resp) { | |
474 | error_setg(errp, "Cannot allocate buffer for identify response"); | |
475 | goto out; | |
476 | } | |
477 | idctrl = (NvmeIdCtrl *)resp; | |
478 | idns = (NvmeIdNs *)resp; | |
479 | r = qemu_vfio_dma_map(s->vfio, resp, sizeof(NvmeIdCtrl), true, &iova); | |
480 | if (r) { | |
481 | error_setg(errp, "Cannot map buffer for DMA"); | |
482 | goto out; | |
483 | } | |
484 | cmd.prp1 = cpu_to_le64(iova); | |
485 | ||
486 | if (nvme_cmd_sync(bs, s->queues[0], &cmd)) { | |
487 | error_setg(errp, "Failed to identify controller"); | |
488 | goto out; | |
489 | } | |
490 | ||
491 | if (le32_to_cpu(idctrl->nn) < namespace) { | |
492 | error_setg(errp, "Invalid namespace"); | |
493 | goto out; | |
494 | } | |
495 | s->write_cache_supported = le32_to_cpu(idctrl->vwc) & 0x1; | |
496 | s->max_transfer = (idctrl->mdts ? 1 << idctrl->mdts : 0) * s->page_size; | |
497 | /* For now the page list buffer per command is one page, to hold at most | |
498 | * s->page_size / sizeof(uint64_t) entries. */ | |
499 | s->max_transfer = MIN_NON_ZERO(s->max_transfer, | |
500 | s->page_size / sizeof(uint64_t) * s->page_size); | |
501 | ||
e0dd95e3 ML |
502 | oncs = le16_to_cpu(idctrl->oncs); |
503 | s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROS); | |
e87a09d6 | 504 | s->supports_discard = !!(oncs & NVME_ONCS_DSM); |
e0dd95e3 | 505 | |
bdd6a90a FZ |
506 | memset(resp, 0, 4096); |
507 | ||
508 | cmd.cdw10 = 0; | |
509 | cmd.nsid = cpu_to_le32(namespace); | |
510 | if (nvme_cmd_sync(bs, s->queues[0], &cmd)) { | |
511 | error_setg(errp, "Failed to identify namespace"); | |
512 | goto out; | |
513 | } | |
514 | ||
515 | s->nsze = le64_to_cpu(idns->nsze); | |
118d1b6a ML |
516 | lbaf = &idns->lbaf[NVME_ID_NS_FLBAS_INDEX(idns->flbas)]; |
517 | ||
e0dd95e3 ML |
518 | if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(idns->dlfeat) && |
519 | NVME_ID_NS_DLFEAT_READ_BEHAVIOR(idns->dlfeat) == | |
520 | NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES) { | |
521 | bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP; | |
522 | } | |
523 | ||
118d1b6a ML |
524 | if (lbaf->ms) { |
525 | error_setg(errp, "Namespaces with metadata are not yet supported"); | |
526 | goto out; | |
527 | } | |
528 | ||
1120407b HR |
529 | if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 || |
530 | (1 << lbaf->ds) > s->page_size) | |
531 | { | |
532 | error_setg(errp, "Namespace has unsupported block size (2^%d)", | |
533 | lbaf->ds); | |
118d1b6a ML |
534 | goto out; |
535 | } | |
bdd6a90a | 536 | |
118d1b6a | 537 | s->blkshift = lbaf->ds; |
bdd6a90a FZ |
538 | out: |
539 | qemu_vfio_dma_unmap(s->vfio, resp); | |
540 | qemu_vfree(resp); | |
541 | } | |
542 | ||
543 | static bool nvme_poll_queues(BDRVNVMeState *s) | |
544 | { | |
545 | bool progress = false; | |
546 | int i; | |
547 | ||
548 | for (i = 0; i < s->nr_queues; i++) { | |
549 | NVMeQueuePair *q = s->queues[i]; | |
2446e0e2 SH |
550 | const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES; |
551 | NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset]; | |
552 | ||
553 | /* | |
554 | * Do an early check for completions. q->lock isn't needed because | |
555 | * nvme_process_completion() only runs in the event loop thread and | |
556 | * cannot race with itself. | |
557 | */ | |
558 | if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) { | |
559 | continue; | |
560 | } | |
561 | ||
bdd6a90a | 562 | qemu_mutex_lock(&q->lock); |
b75fd5f5 | 563 | while (nvme_process_completion(q)) { |
bdd6a90a FZ |
564 | /* Keep polling */ |
565 | progress = true; | |
566 | } | |
567 | qemu_mutex_unlock(&q->lock); | |
568 | } | |
569 | return progress; | |
570 | } | |
571 | ||
572 | static void nvme_handle_event(EventNotifier *n) | |
573 | { | |
574 | BDRVNVMeState *s = container_of(n, BDRVNVMeState, irq_notifier); | |
575 | ||
576 | trace_nvme_handle_event(s); | |
bdd6a90a FZ |
577 | event_notifier_test_and_clear(n); |
578 | nvme_poll_queues(s); | |
bdd6a90a FZ |
579 | } |
580 | ||
581 | static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp) | |
582 | { | |
583 | BDRVNVMeState *s = bs->opaque; | |
584 | int n = s->nr_queues; | |
585 | NVMeQueuePair *q; | |
586 | NvmeCmd cmd; | |
587 | int queue_size = NVME_QUEUE_SIZE; | |
588 | ||
589 | q = nvme_create_queue_pair(bs, n, queue_size, errp); | |
590 | if (!q) { | |
591 | return false; | |
592 | } | |
593 | cmd = (NvmeCmd) { | |
594 | .opcode = NVME_ADM_CMD_CREATE_CQ, | |
595 | .prp1 = cpu_to_le64(q->cq.iova), | |
596 | .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)), | |
597 | .cdw11 = cpu_to_le32(0x3), | |
598 | }; | |
599 | if (nvme_cmd_sync(bs, s->queues[0], &cmd)) { | |
600 | error_setg(errp, "Failed to create io queue [%d]", n); | |
b75fd5f5 | 601 | nvme_free_queue_pair(q); |
bdd6a90a FZ |
602 | return false; |
603 | } | |
604 | cmd = (NvmeCmd) { | |
605 | .opcode = NVME_ADM_CMD_CREATE_SQ, | |
606 | .prp1 = cpu_to_le64(q->sq.iova), | |
607 | .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)), | |
608 | .cdw11 = cpu_to_le32(0x1 | (n << 16)), | |
609 | }; | |
610 | if (nvme_cmd_sync(bs, s->queues[0], &cmd)) { | |
611 | error_setg(errp, "Failed to create io queue [%d]", n); | |
b75fd5f5 | 612 | nvme_free_queue_pair(q); |
bdd6a90a FZ |
613 | return false; |
614 | } | |
615 | s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1); | |
616 | s->queues[n] = q; | |
617 | s->nr_queues++; | |
618 | return true; | |
619 | } | |
620 | ||
621 | static bool nvme_poll_cb(void *opaque) | |
622 | { | |
623 | EventNotifier *e = opaque; | |
624 | BDRVNVMeState *s = container_of(e, BDRVNVMeState, irq_notifier); | |
bdd6a90a FZ |
625 | |
626 | trace_nvme_poll_cb(s); | |
b3ac2b94 | 627 | return nvme_poll_queues(s); |
bdd6a90a FZ |
628 | } |
629 | ||
630 | static int nvme_init(BlockDriverState *bs, const char *device, int namespace, | |
631 | Error **errp) | |
632 | { | |
633 | BDRVNVMeState *s = bs->opaque; | |
634 | int ret; | |
635 | uint64_t cap; | |
636 | uint64_t timeout_ms; | |
637 | uint64_t deadline, now; | |
638 | Error *local_err = NULL; | |
639 | ||
640 | qemu_co_mutex_init(&s->dma_map_lock); | |
641 | qemu_co_queue_init(&s->dma_flush_queue); | |
cc61b074 | 642 | s->device = g_strdup(device); |
bdd6a90a FZ |
643 | s->nsid = namespace; |
644 | s->aio_context = bdrv_get_aio_context(bs); | |
645 | ret = event_notifier_init(&s->irq_notifier, 0); | |
646 | if (ret) { | |
647 | error_setg(errp, "Failed to init event notifier"); | |
648 | return ret; | |
649 | } | |
650 | ||
651 | s->vfio = qemu_vfio_open_pci(device, errp); | |
652 | if (!s->vfio) { | |
653 | ret = -EINVAL; | |
9582f357 | 654 | goto out; |
bdd6a90a FZ |
655 | } |
656 | ||
657 | s->regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, NVME_BAR_SIZE, errp); | |
658 | if (!s->regs) { | |
659 | ret = -EINVAL; | |
9582f357 | 660 | goto out; |
bdd6a90a FZ |
661 | } |
662 | ||
663 | /* Perform initialize sequence as described in NVMe spec "7.6.1 | |
664 | * Initialization". */ | |
665 | ||
666 | cap = le64_to_cpu(s->regs->cap); | |
667 | if (!(cap & (1ULL << 37))) { | |
668 | error_setg(errp, "Device doesn't support NVMe command set"); | |
669 | ret = -EINVAL; | |
9582f357 | 670 | goto out; |
bdd6a90a FZ |
671 | } |
672 | ||
673 | s->page_size = MAX(4096, 1 << (12 + ((cap >> 48) & 0xF))); | |
674 | s->doorbell_scale = (4 << (((cap >> 32) & 0xF))) / sizeof(uint32_t); | |
675 | bs->bl.opt_mem_alignment = s->page_size; | |
676 | timeout_ms = MIN(500 * ((cap >> 24) & 0xFF), 30000); | |
677 | ||
678 | /* Reset device to get a clean state. */ | |
679 | s->regs->cc = cpu_to_le32(le32_to_cpu(s->regs->cc) & 0xFE); | |
680 | /* Wait for CSTS.RDY = 0. */ | |
681 | deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * 1000000ULL; | |
682 | while (le32_to_cpu(s->regs->csts) & 0x1) { | |
683 | if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) { | |
684 | error_setg(errp, "Timeout while waiting for device to reset (%" | |
685 | PRId64 " ms)", | |
686 | timeout_ms); | |
687 | ret = -ETIMEDOUT; | |
9582f357 | 688 | goto out; |
bdd6a90a FZ |
689 | } |
690 | } | |
691 | ||
692 | /* Set up admin queue. */ | |
693 | s->queues = g_new(NVMeQueuePair *, 1); | |
bdd6a90a FZ |
694 | s->queues[0] = nvme_create_queue_pair(bs, 0, NVME_QUEUE_SIZE, errp); |
695 | if (!s->queues[0]) { | |
696 | ret = -EINVAL; | |
9582f357 | 697 | goto out; |
bdd6a90a | 698 | } |
95667c3b | 699 | s->nr_queues = 1; |
bdd6a90a FZ |
700 | QEMU_BUILD_BUG_ON(NVME_QUEUE_SIZE & 0xF000); |
701 | s->regs->aqa = cpu_to_le32((NVME_QUEUE_SIZE << 16) | NVME_QUEUE_SIZE); | |
702 | s->regs->asq = cpu_to_le64(s->queues[0]->sq.iova); | |
703 | s->regs->acq = cpu_to_le64(s->queues[0]->cq.iova); | |
704 | ||
705 | /* After setting up all control registers we can enable device now. */ | |
706 | s->regs->cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << 20) | | |
707 | (ctz32(NVME_SQ_ENTRY_BYTES) << 16) | | |
708 | 0x1); | |
709 | /* Wait for CSTS.RDY = 1. */ | |
710 | now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); | |
711 | deadline = now + timeout_ms * 1000000; | |
712 | while (!(le32_to_cpu(s->regs->csts) & 0x1)) { | |
713 | if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) { | |
714 | error_setg(errp, "Timeout while waiting for device to start (%" | |
715 | PRId64 " ms)", | |
716 | timeout_ms); | |
717 | ret = -ETIMEDOUT; | |
9582f357 | 718 | goto out; |
bdd6a90a FZ |
719 | } |
720 | } | |
721 | ||
722 | ret = qemu_vfio_pci_init_irq(s->vfio, &s->irq_notifier, | |
723 | VFIO_PCI_MSIX_IRQ_INDEX, errp); | |
724 | if (ret) { | |
9582f357 | 725 | goto out; |
bdd6a90a FZ |
726 | } |
727 | aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier, | |
728 | false, nvme_handle_event, nvme_poll_cb); | |
729 | ||
78d8c99e | 730 | nvme_identify(bs, namespace, &local_err); |
bdd6a90a FZ |
731 | if (local_err) { |
732 | error_propagate(errp, local_err); | |
733 | ret = -EIO; | |
9582f357 | 734 | goto out; |
bdd6a90a FZ |
735 | } |
736 | ||
737 | /* Set up command queues. */ | |
738 | if (!nvme_add_io_queue(bs, errp)) { | |
739 | ret = -EIO; | |
bdd6a90a | 740 | } |
9582f357 FZ |
741 | out: |
742 | /* Cleaning up is done in nvme_file_open() upon error. */ | |
bdd6a90a FZ |
743 | return ret; |
744 | } | |
745 | ||
746 | /* Parse a filename in the format of nvme://XXXX:XX:XX.X/X. Example: | |
747 | * | |
748 | * nvme://0000:44:00.0/1 | |
749 | * | |
750 | * where the "nvme://" is a fixed form of the protocol prefix, the middle part | |
751 | * is the PCI address, and the last part is the namespace number starting from | |
752 | * 1 according to the NVMe spec. */ | |
753 | static void nvme_parse_filename(const char *filename, QDict *options, | |
754 | Error **errp) | |
755 | { | |
756 | int pref = strlen("nvme://"); | |
757 | ||
758 | if (strlen(filename) > pref && !strncmp(filename, "nvme://", pref)) { | |
759 | const char *tmp = filename + pref; | |
760 | char *device; | |
761 | const char *namespace; | |
762 | unsigned long ns; | |
763 | const char *slash = strchr(tmp, '/'); | |
764 | if (!slash) { | |
625eaca9 | 765 | qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp); |
bdd6a90a FZ |
766 | return; |
767 | } | |
768 | device = g_strndup(tmp, slash - tmp); | |
625eaca9 | 769 | qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device); |
bdd6a90a FZ |
770 | g_free(device); |
771 | namespace = slash + 1; | |
772 | if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) { | |
773 | error_setg(errp, "Invalid namespace '%s', positive number expected", | |
774 | namespace); | |
775 | return; | |
776 | } | |
625eaca9 LV |
777 | qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE, |
778 | *namespace ? namespace : "1"); | |
bdd6a90a FZ |
779 | } |
780 | } | |
781 | ||
782 | static int nvme_enable_disable_write_cache(BlockDriverState *bs, bool enable, | |
783 | Error **errp) | |
784 | { | |
785 | int ret; | |
786 | BDRVNVMeState *s = bs->opaque; | |
787 | NvmeCmd cmd = { | |
788 | .opcode = NVME_ADM_CMD_SET_FEATURES, | |
789 | .nsid = cpu_to_le32(s->nsid), | |
790 | .cdw10 = cpu_to_le32(0x06), | |
791 | .cdw11 = cpu_to_le32(enable ? 0x01 : 0x00), | |
792 | }; | |
793 | ||
794 | ret = nvme_cmd_sync(bs, s->queues[0], &cmd); | |
795 | if (ret) { | |
796 | error_setg(errp, "Failed to configure NVMe write cache"); | |
797 | } | |
798 | return ret; | |
799 | } | |
800 | ||
801 | static void nvme_close(BlockDriverState *bs) | |
802 | { | |
803 | int i; | |
804 | BDRVNVMeState *s = bs->opaque; | |
805 | ||
806 | for (i = 0; i < s->nr_queues; ++i) { | |
b75fd5f5 | 807 | nvme_free_queue_pair(s->queues[i]); |
bdd6a90a | 808 | } |
9582f357 | 809 | g_free(s->queues); |
bdd6a90a FZ |
810 | aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier, |
811 | false, NULL, NULL); | |
9582f357 | 812 | event_notifier_cleanup(&s->irq_notifier); |
bdd6a90a FZ |
813 | qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->regs, 0, NVME_BAR_SIZE); |
814 | qemu_vfio_close(s->vfio); | |
cc61b074 HR |
815 | |
816 | g_free(s->device); | |
bdd6a90a FZ |
817 | } |
818 | ||
819 | static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags, | |
820 | Error **errp) | |
821 | { | |
822 | const char *device; | |
823 | QemuOpts *opts; | |
824 | int namespace; | |
825 | int ret; | |
826 | BDRVNVMeState *s = bs->opaque; | |
827 | ||
e0dd95e3 ML |
828 | bs->supported_write_flags = BDRV_REQ_FUA; |
829 | ||
bdd6a90a FZ |
830 | opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort); |
831 | qemu_opts_absorb_qdict(opts, options, &error_abort); | |
832 | device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE); | |
833 | if (!device) { | |
834 | error_setg(errp, "'" NVME_BLOCK_OPT_DEVICE "' option is required"); | |
835 | qemu_opts_del(opts); | |
836 | return -EINVAL; | |
837 | } | |
838 | ||
839 | namespace = qemu_opt_get_number(opts, NVME_BLOCK_OPT_NAMESPACE, 1); | |
840 | ret = nvme_init(bs, device, namespace, errp); | |
841 | qemu_opts_del(opts); | |
842 | if (ret) { | |
843 | goto fail; | |
844 | } | |
845 | if (flags & BDRV_O_NOCACHE) { | |
846 | if (!s->write_cache_supported) { | |
847 | error_setg(errp, | |
848 | "NVMe controller doesn't support write cache configuration"); | |
849 | ret = -EINVAL; | |
850 | } else { | |
851 | ret = nvme_enable_disable_write_cache(bs, !(flags & BDRV_O_NOCACHE), | |
852 | errp); | |
853 | } | |
854 | if (ret) { | |
855 | goto fail; | |
856 | } | |
857 | } | |
bdd6a90a FZ |
858 | return 0; |
859 | fail: | |
860 | nvme_close(bs); | |
861 | return ret; | |
862 | } | |
863 | ||
864 | static int64_t nvme_getlength(BlockDriverState *bs) | |
865 | { | |
866 | BDRVNVMeState *s = bs->opaque; | |
118d1b6a ML |
867 | return s->nsze << s->blkshift; |
868 | } | |
bdd6a90a | 869 | |
1120407b | 870 | static uint32_t nvme_get_blocksize(BlockDriverState *bs) |
118d1b6a ML |
871 | { |
872 | BDRVNVMeState *s = bs->opaque; | |
1120407b HR |
873 | assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12); |
874 | return UINT32_C(1) << s->blkshift; | |
118d1b6a ML |
875 | } |
876 | ||
877 | static int nvme_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz) | |
878 | { | |
1120407b | 879 | uint32_t blocksize = nvme_get_blocksize(bs); |
118d1b6a ML |
880 | bsz->phys = blocksize; |
881 | bsz->log = blocksize; | |
882 | return 0; | |
bdd6a90a FZ |
883 | } |
884 | ||
885 | /* Called with s->dma_map_lock */ | |
886 | static coroutine_fn int nvme_cmd_unmap_qiov(BlockDriverState *bs, | |
887 | QEMUIOVector *qiov) | |
888 | { | |
889 | int r = 0; | |
890 | BDRVNVMeState *s = bs->opaque; | |
891 | ||
892 | s->dma_map_count -= qiov->size; | |
893 | if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) { | |
894 | r = qemu_vfio_dma_reset_temporary(s->vfio); | |
895 | if (!r) { | |
896 | qemu_co_queue_restart_all(&s->dma_flush_queue); | |
897 | } | |
898 | } | |
899 | return r; | |
900 | } | |
901 | ||
902 | /* Called with s->dma_map_lock */ | |
903 | static coroutine_fn int nvme_cmd_map_qiov(BlockDriverState *bs, NvmeCmd *cmd, | |
904 | NVMeRequest *req, QEMUIOVector *qiov) | |
905 | { | |
906 | BDRVNVMeState *s = bs->opaque; | |
907 | uint64_t *pagelist = req->prp_list_page; | |
908 | int i, j, r; | |
909 | int entries = 0; | |
910 | ||
911 | assert(qiov->size); | |
912 | assert(QEMU_IS_ALIGNED(qiov->size, s->page_size)); | |
913 | assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t)); | |
914 | for (i = 0; i < qiov->niov; ++i) { | |
915 | bool retry = true; | |
916 | uint64_t iova; | |
917 | try_map: | |
918 | r = qemu_vfio_dma_map(s->vfio, | |
919 | qiov->iov[i].iov_base, | |
920 | qiov->iov[i].iov_len, | |
921 | true, &iova); | |
922 | if (r == -ENOMEM && retry) { | |
923 | retry = false; | |
924 | trace_nvme_dma_flush_queue_wait(s); | |
925 | if (s->dma_map_count) { | |
926 | trace_nvme_dma_map_flush(s); | |
927 | qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock); | |
928 | } else { | |
929 | r = qemu_vfio_dma_reset_temporary(s->vfio); | |
930 | if (r) { | |
931 | goto fail; | |
932 | } | |
933 | } | |
934 | goto try_map; | |
935 | } | |
936 | if (r) { | |
937 | goto fail; | |
938 | } | |
939 | ||
940 | for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) { | |
2916405a | 941 | pagelist[entries++] = cpu_to_le64(iova + j * s->page_size); |
bdd6a90a FZ |
942 | } |
943 | trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base, | |
944 | qiov->iov[i].iov_len / s->page_size); | |
945 | } | |
946 | ||
947 | s->dma_map_count += qiov->size; | |
948 | ||
949 | assert(entries <= s->page_size / sizeof(uint64_t)); | |
950 | switch (entries) { | |
951 | case 0: | |
952 | abort(); | |
953 | case 1: | |
2916405a | 954 | cmd->prp1 = pagelist[0]; |
bdd6a90a FZ |
955 | cmd->prp2 = 0; |
956 | break; | |
957 | case 2: | |
2916405a LF |
958 | cmd->prp1 = pagelist[0]; |
959 | cmd->prp2 = pagelist[1]; | |
bdd6a90a FZ |
960 | break; |
961 | default: | |
2916405a LF |
962 | cmd->prp1 = pagelist[0]; |
963 | cmd->prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t)); | |
bdd6a90a FZ |
964 | break; |
965 | } | |
966 | trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries); | |
967 | for (i = 0; i < entries; ++i) { | |
968 | trace_nvme_cmd_map_qiov_pages(s, i, pagelist[i]); | |
969 | } | |
970 | return 0; | |
971 | fail: | |
972 | /* No need to unmap [0 - i) iovs even if we've failed, since we don't | |
973 | * increment s->dma_map_count. This is okay for fixed mapping memory areas | |
974 | * because they are already mapped before calling this function; for | |
975 | * temporary mappings, a later nvme_cmd_(un)map_qiov will reclaim by | |
976 | * calling qemu_vfio_dma_reset_temporary when necessary. */ | |
977 | return r; | |
978 | } | |
979 | ||
980 | typedef struct { | |
981 | Coroutine *co; | |
982 | int ret; | |
983 | AioContext *ctx; | |
984 | } NVMeCoData; | |
985 | ||
986 | static void nvme_rw_cb_bh(void *opaque) | |
987 | { | |
988 | NVMeCoData *data = opaque; | |
989 | qemu_coroutine_enter(data->co); | |
990 | } | |
991 | ||
992 | static void nvme_rw_cb(void *opaque, int ret) | |
993 | { | |
994 | NVMeCoData *data = opaque; | |
995 | data->ret = ret; | |
996 | if (!data->co) { | |
997 | /* The rw coroutine hasn't yielded, don't try to enter. */ | |
998 | return; | |
999 | } | |
e4ec5ad4 | 1000 | replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data); |
bdd6a90a FZ |
1001 | } |
1002 | ||
1003 | static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs, | |
1004 | uint64_t offset, uint64_t bytes, | |
1005 | QEMUIOVector *qiov, | |
1006 | bool is_write, | |
1007 | int flags) | |
1008 | { | |
1009 | int r; | |
1010 | BDRVNVMeState *s = bs->opaque; | |
1011 | NVMeQueuePair *ioq = s->queues[1]; | |
1012 | NVMeRequest *req; | |
118d1b6a ML |
1013 | |
1014 | uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) | | |
bdd6a90a FZ |
1015 | (flags & BDRV_REQ_FUA ? 1 << 30 : 0); |
1016 | NvmeCmd cmd = { | |
1017 | .opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ, | |
1018 | .nsid = cpu_to_le32(s->nsid), | |
118d1b6a ML |
1019 | .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF), |
1020 | .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF), | |
bdd6a90a FZ |
1021 | .cdw12 = cpu_to_le32(cdw12), |
1022 | }; | |
1023 | NVMeCoData data = { | |
1024 | .ctx = bdrv_get_aio_context(bs), | |
1025 | .ret = -EINPROGRESS, | |
1026 | }; | |
1027 | ||
1028 | trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov); | |
1029 | assert(s->nr_queues > 1); | |
1030 | req = nvme_get_free_req(ioq); | |
1031 | assert(req); | |
1032 | ||
1033 | qemu_co_mutex_lock(&s->dma_map_lock); | |
1034 | r = nvme_cmd_map_qiov(bs, &cmd, req, qiov); | |
1035 | qemu_co_mutex_unlock(&s->dma_map_lock); | |
1036 | if (r) { | |
b75fd5f5 | 1037 | nvme_put_free_req_and_wake(ioq, req); |
bdd6a90a FZ |
1038 | return r; |
1039 | } | |
b75fd5f5 | 1040 | nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data); |
bdd6a90a FZ |
1041 | |
1042 | data.co = qemu_coroutine_self(); | |
1043 | while (data.ret == -EINPROGRESS) { | |
1044 | qemu_coroutine_yield(); | |
1045 | } | |
1046 | ||
1047 | qemu_co_mutex_lock(&s->dma_map_lock); | |
1048 | r = nvme_cmd_unmap_qiov(bs, qiov); | |
1049 | qemu_co_mutex_unlock(&s->dma_map_lock); | |
1050 | if (r) { | |
1051 | return r; | |
1052 | } | |
1053 | ||
1054 | trace_nvme_rw_done(s, is_write, offset, bytes, data.ret); | |
1055 | return data.ret; | |
1056 | } | |
1057 | ||
1058 | static inline bool nvme_qiov_aligned(BlockDriverState *bs, | |
1059 | const QEMUIOVector *qiov) | |
1060 | { | |
1061 | int i; | |
1062 | BDRVNVMeState *s = bs->opaque; | |
1063 | ||
1064 | for (i = 0; i < qiov->niov; ++i) { | |
1065 | if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, s->page_size) || | |
1066 | !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, s->page_size)) { | |
1067 | trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base, | |
1068 | qiov->iov[i].iov_len, s->page_size); | |
1069 | return false; | |
1070 | } | |
1071 | } | |
1072 | return true; | |
1073 | } | |
1074 | ||
1075 | static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes, | |
1076 | QEMUIOVector *qiov, bool is_write, int flags) | |
1077 | { | |
1078 | BDRVNVMeState *s = bs->opaque; | |
1079 | int r; | |
1080 | uint8_t *buf = NULL; | |
1081 | QEMUIOVector local_qiov; | |
1082 | ||
1083 | assert(QEMU_IS_ALIGNED(offset, s->page_size)); | |
1084 | assert(QEMU_IS_ALIGNED(bytes, s->page_size)); | |
1085 | assert(bytes <= s->max_transfer); | |
1086 | if (nvme_qiov_aligned(bs, qiov)) { | |
1087 | return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags); | |
1088 | } | |
1089 | trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write); | |
1090 | buf = qemu_try_blockalign(bs, bytes); | |
1091 | ||
1092 | if (!buf) { | |
1093 | return -ENOMEM; | |
1094 | } | |
1095 | qemu_iovec_init(&local_qiov, 1); | |
1096 | if (is_write) { | |
1097 | qemu_iovec_to_buf(qiov, 0, buf, bytes); | |
1098 | } | |
1099 | qemu_iovec_add(&local_qiov, buf, bytes); | |
1100 | r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags); | |
1101 | qemu_iovec_destroy(&local_qiov); | |
1102 | if (!r && !is_write) { | |
1103 | qemu_iovec_from_buf(qiov, 0, buf, bytes); | |
1104 | } | |
1105 | qemu_vfree(buf); | |
1106 | return r; | |
1107 | } | |
1108 | ||
1109 | static coroutine_fn int nvme_co_preadv(BlockDriverState *bs, | |
1110 | uint64_t offset, uint64_t bytes, | |
1111 | QEMUIOVector *qiov, int flags) | |
1112 | { | |
1113 | return nvme_co_prw(bs, offset, bytes, qiov, false, flags); | |
1114 | } | |
1115 | ||
1116 | static coroutine_fn int nvme_co_pwritev(BlockDriverState *bs, | |
1117 | uint64_t offset, uint64_t bytes, | |
1118 | QEMUIOVector *qiov, int flags) | |
1119 | { | |
1120 | return nvme_co_prw(bs, offset, bytes, qiov, true, flags); | |
1121 | } | |
1122 | ||
1123 | static coroutine_fn int nvme_co_flush(BlockDriverState *bs) | |
1124 | { | |
1125 | BDRVNVMeState *s = bs->opaque; | |
1126 | NVMeQueuePair *ioq = s->queues[1]; | |
1127 | NVMeRequest *req; | |
1128 | NvmeCmd cmd = { | |
1129 | .opcode = NVME_CMD_FLUSH, | |
1130 | .nsid = cpu_to_le32(s->nsid), | |
1131 | }; | |
1132 | NVMeCoData data = { | |
1133 | .ctx = bdrv_get_aio_context(bs), | |
1134 | .ret = -EINPROGRESS, | |
1135 | }; | |
1136 | ||
1137 | assert(s->nr_queues > 1); | |
1138 | req = nvme_get_free_req(ioq); | |
1139 | assert(req); | |
b75fd5f5 | 1140 | nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data); |
bdd6a90a FZ |
1141 | |
1142 | data.co = qemu_coroutine_self(); | |
1143 | if (data.ret == -EINPROGRESS) { | |
1144 | qemu_coroutine_yield(); | |
1145 | } | |
1146 | ||
1147 | return data.ret; | |
1148 | } | |
1149 | ||
1150 | ||
e0dd95e3 ML |
1151 | static coroutine_fn int nvme_co_pwrite_zeroes(BlockDriverState *bs, |
1152 | int64_t offset, | |
1153 | int bytes, | |
1154 | BdrvRequestFlags flags) | |
1155 | { | |
1156 | BDRVNVMeState *s = bs->opaque; | |
1157 | NVMeQueuePair *ioq = s->queues[1]; | |
1158 | NVMeRequest *req; | |
1159 | ||
1160 | uint32_t cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF; | |
1161 | ||
1162 | if (!s->supports_write_zeroes) { | |
1163 | return -ENOTSUP; | |
1164 | } | |
1165 | ||
1166 | NvmeCmd cmd = { | |
1167 | .opcode = NVME_CMD_WRITE_ZEROS, | |
1168 | .nsid = cpu_to_le32(s->nsid), | |
1169 | .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF), | |
1170 | .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF), | |
1171 | }; | |
1172 | ||
1173 | NVMeCoData data = { | |
1174 | .ctx = bdrv_get_aio_context(bs), | |
1175 | .ret = -EINPROGRESS, | |
1176 | }; | |
1177 | ||
1178 | if (flags & BDRV_REQ_MAY_UNMAP) { | |
1179 | cdw12 |= (1 << 25); | |
1180 | } | |
1181 | ||
1182 | if (flags & BDRV_REQ_FUA) { | |
1183 | cdw12 |= (1 << 30); | |
1184 | } | |
1185 | ||
1186 | cmd.cdw12 = cpu_to_le32(cdw12); | |
1187 | ||
1188 | trace_nvme_write_zeroes(s, offset, bytes, flags); | |
1189 | assert(s->nr_queues > 1); | |
1190 | req = nvme_get_free_req(ioq); | |
1191 | assert(req); | |
1192 | ||
b75fd5f5 | 1193 | nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data); |
e0dd95e3 ML |
1194 | |
1195 | data.co = qemu_coroutine_self(); | |
1196 | while (data.ret == -EINPROGRESS) { | |
1197 | qemu_coroutine_yield(); | |
1198 | } | |
1199 | ||
1200 | trace_nvme_rw_done(s, true, offset, bytes, data.ret); | |
1201 | return data.ret; | |
1202 | } | |
1203 | ||
1204 | ||
e87a09d6 ML |
1205 | static int coroutine_fn nvme_co_pdiscard(BlockDriverState *bs, |
1206 | int64_t offset, | |
1207 | int bytes) | |
1208 | { | |
1209 | BDRVNVMeState *s = bs->opaque; | |
1210 | NVMeQueuePair *ioq = s->queues[1]; | |
1211 | NVMeRequest *req; | |
1212 | NvmeDsmRange *buf; | |
1213 | QEMUIOVector local_qiov; | |
1214 | int ret; | |
1215 | ||
1216 | NvmeCmd cmd = { | |
1217 | .opcode = NVME_CMD_DSM, | |
1218 | .nsid = cpu_to_le32(s->nsid), | |
1219 | .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/ | |
1220 | .cdw11 = cpu_to_le32(1 << 2), /*deallocate bit*/ | |
1221 | }; | |
1222 | ||
1223 | NVMeCoData data = { | |
1224 | .ctx = bdrv_get_aio_context(bs), | |
1225 | .ret = -EINPROGRESS, | |
1226 | }; | |
1227 | ||
1228 | if (!s->supports_discard) { | |
1229 | return -ENOTSUP; | |
1230 | } | |
1231 | ||
1232 | assert(s->nr_queues > 1); | |
1233 | ||
1234 | buf = qemu_try_blockalign0(bs, s->page_size); | |
1235 | if (!buf) { | |
1236 | return -ENOMEM; | |
1237 | } | |
1238 | ||
1239 | buf->nlb = cpu_to_le32(bytes >> s->blkshift); | |
1240 | buf->slba = cpu_to_le64(offset >> s->blkshift); | |
1241 | buf->cattr = 0; | |
1242 | ||
1243 | qemu_iovec_init(&local_qiov, 1); | |
1244 | qemu_iovec_add(&local_qiov, buf, 4096); | |
1245 | ||
1246 | req = nvme_get_free_req(ioq); | |
1247 | assert(req); | |
1248 | ||
1249 | qemu_co_mutex_lock(&s->dma_map_lock); | |
1250 | ret = nvme_cmd_map_qiov(bs, &cmd, req, &local_qiov); | |
1251 | qemu_co_mutex_unlock(&s->dma_map_lock); | |
1252 | ||
1253 | if (ret) { | |
b75fd5f5 | 1254 | nvme_put_free_req_and_wake(ioq, req); |
e87a09d6 ML |
1255 | goto out; |
1256 | } | |
1257 | ||
1258 | trace_nvme_dsm(s, offset, bytes); | |
1259 | ||
b75fd5f5 | 1260 | nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data); |
e87a09d6 ML |
1261 | |
1262 | data.co = qemu_coroutine_self(); | |
1263 | while (data.ret == -EINPROGRESS) { | |
1264 | qemu_coroutine_yield(); | |
1265 | } | |
1266 | ||
1267 | qemu_co_mutex_lock(&s->dma_map_lock); | |
1268 | ret = nvme_cmd_unmap_qiov(bs, &local_qiov); | |
1269 | qemu_co_mutex_unlock(&s->dma_map_lock); | |
1270 | ||
1271 | if (ret) { | |
1272 | goto out; | |
1273 | } | |
1274 | ||
1275 | ret = data.ret; | |
1276 | trace_nvme_dsm_done(s, offset, bytes, ret); | |
1277 | out: | |
1278 | qemu_iovec_destroy(&local_qiov); | |
1279 | qemu_vfree(buf); | |
1280 | return ret; | |
1281 | ||
1282 | } | |
1283 | ||
1284 | ||
bdd6a90a FZ |
1285 | static int nvme_reopen_prepare(BDRVReopenState *reopen_state, |
1286 | BlockReopenQueue *queue, Error **errp) | |
1287 | { | |
1288 | return 0; | |
1289 | } | |
1290 | ||
998b3a1e | 1291 | static void nvme_refresh_filename(BlockDriverState *bs) |
bdd6a90a | 1292 | { |
cc61b074 | 1293 | BDRVNVMeState *s = bs->opaque; |
bdd6a90a | 1294 | |
cc61b074 HR |
1295 | snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i", |
1296 | s->device, s->nsid); | |
bdd6a90a FZ |
1297 | } |
1298 | ||
1299 | static void nvme_refresh_limits(BlockDriverState *bs, Error **errp) | |
1300 | { | |
1301 | BDRVNVMeState *s = bs->opaque; | |
1302 | ||
1303 | bs->bl.opt_mem_alignment = s->page_size; | |
1304 | bs->bl.request_alignment = s->page_size; | |
1305 | bs->bl.max_transfer = s->max_transfer; | |
1306 | } | |
1307 | ||
1308 | static void nvme_detach_aio_context(BlockDriverState *bs) | |
1309 | { | |
1310 | BDRVNVMeState *s = bs->opaque; | |
1311 | ||
1312 | aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier, | |
1313 | false, NULL, NULL); | |
1314 | } | |
1315 | ||
1316 | static void nvme_attach_aio_context(BlockDriverState *bs, | |
1317 | AioContext *new_context) | |
1318 | { | |
1319 | BDRVNVMeState *s = bs->opaque; | |
1320 | ||
1321 | s->aio_context = new_context; | |
1322 | aio_set_event_notifier(new_context, &s->irq_notifier, | |
1323 | false, nvme_handle_event, nvme_poll_cb); | |
1324 | } | |
1325 | ||
1326 | static void nvme_aio_plug(BlockDriverState *bs) | |
1327 | { | |
1328 | BDRVNVMeState *s = bs->opaque; | |
2f0d8947 PB |
1329 | assert(!s->plugged); |
1330 | s->plugged = true; | |
bdd6a90a FZ |
1331 | } |
1332 | ||
1333 | static void nvme_aio_unplug(BlockDriverState *bs) | |
1334 | { | |
1335 | int i; | |
1336 | BDRVNVMeState *s = bs->opaque; | |
1337 | assert(s->plugged); | |
2f0d8947 PB |
1338 | s->plugged = false; |
1339 | for (i = 1; i < s->nr_queues; i++) { | |
1340 | NVMeQueuePair *q = s->queues[i]; | |
1341 | qemu_mutex_lock(&q->lock); | |
b75fd5f5 SH |
1342 | nvme_kick(q); |
1343 | nvme_process_completion(q); | |
2f0d8947 | 1344 | qemu_mutex_unlock(&q->lock); |
bdd6a90a FZ |
1345 | } |
1346 | } | |
1347 | ||
9ed61612 FZ |
1348 | static void nvme_register_buf(BlockDriverState *bs, void *host, size_t size) |
1349 | { | |
1350 | int ret; | |
1351 | BDRVNVMeState *s = bs->opaque; | |
1352 | ||
1353 | ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL); | |
1354 | if (ret) { | |
1355 | /* FIXME: we may run out of IOVA addresses after repeated | |
1356 | * bdrv_register_buf/bdrv_unregister_buf, because nvme_vfio_dma_unmap | |
1357 | * doesn't reclaim addresses for fixed mappings. */ | |
1358 | error_report("nvme_register_buf failed: %s", strerror(-ret)); | |
1359 | } | |
1360 | } | |
1361 | ||
1362 | static void nvme_unregister_buf(BlockDriverState *bs, void *host) | |
1363 | { | |
1364 | BDRVNVMeState *s = bs->opaque; | |
1365 | ||
1366 | qemu_vfio_dma_unmap(s->vfio, host); | |
1367 | } | |
1368 | ||
2654267c HR |
1369 | static const char *const nvme_strong_runtime_opts[] = { |
1370 | NVME_BLOCK_OPT_DEVICE, | |
1371 | NVME_BLOCK_OPT_NAMESPACE, | |
1372 | ||
1373 | NULL | |
1374 | }; | |
1375 | ||
bdd6a90a FZ |
1376 | static BlockDriver bdrv_nvme = { |
1377 | .format_name = "nvme", | |
1378 | .protocol_name = "nvme", | |
1379 | .instance_size = sizeof(BDRVNVMeState), | |
1380 | ||
5a5e7f8c ML |
1381 | .bdrv_co_create_opts = bdrv_co_create_opts_simple, |
1382 | .create_opts = &bdrv_create_opts_simple, | |
1383 | ||
bdd6a90a FZ |
1384 | .bdrv_parse_filename = nvme_parse_filename, |
1385 | .bdrv_file_open = nvme_file_open, | |
1386 | .bdrv_close = nvme_close, | |
1387 | .bdrv_getlength = nvme_getlength, | |
118d1b6a | 1388 | .bdrv_probe_blocksizes = nvme_probe_blocksizes, |
bdd6a90a FZ |
1389 | |
1390 | .bdrv_co_preadv = nvme_co_preadv, | |
1391 | .bdrv_co_pwritev = nvme_co_pwritev, | |
e0dd95e3 ML |
1392 | |
1393 | .bdrv_co_pwrite_zeroes = nvme_co_pwrite_zeroes, | |
e87a09d6 | 1394 | .bdrv_co_pdiscard = nvme_co_pdiscard, |
e0dd95e3 | 1395 | |
bdd6a90a FZ |
1396 | .bdrv_co_flush_to_disk = nvme_co_flush, |
1397 | .bdrv_reopen_prepare = nvme_reopen_prepare, | |
1398 | ||
bdd6a90a FZ |
1399 | .bdrv_refresh_filename = nvme_refresh_filename, |
1400 | .bdrv_refresh_limits = nvme_refresh_limits, | |
2654267c | 1401 | .strong_runtime_opts = nvme_strong_runtime_opts, |
bdd6a90a FZ |
1402 | |
1403 | .bdrv_detach_aio_context = nvme_detach_aio_context, | |
1404 | .bdrv_attach_aio_context = nvme_attach_aio_context, | |
1405 | ||
1406 | .bdrv_io_plug = nvme_aio_plug, | |
1407 | .bdrv_io_unplug = nvme_aio_unplug, | |
9ed61612 FZ |
1408 | |
1409 | .bdrv_register_buf = nvme_register_buf, | |
1410 | .bdrv_unregister_buf = nvme_unregister_buf, | |
bdd6a90a FZ |
1411 | }; |
1412 | ||
1413 | static void bdrv_nvme_init(void) | |
1414 | { | |
1415 | bdrv_register(&bdrv_nvme); | |
1416 | } | |
1417 | ||
1418 | block_init(bdrv_nvme_init); |