2 * Inter-VM Shared Memory PCI device.
5 * Cam Macdonell <cam@cs.ualberta.ca>
7 * Based On: cirrus_vga.c
8 * Copyright (c) 2004 Fabrice Bellard
9 * Copyright (c) 2004 Makoto Suzuki (suzu)
12 * Copyright (c) 2006 Igor Kovalenko
14 * This code is licensed under the GNU GPL v2.
16 * Contributions after 2012-01-13 are licensed under the terms of the
17 * GNU GPL, version 2 or (at your option) any later version.
24 #include "migration.h"
28 #include <sys/types.h>
30 #define IVSHMEM_IOEVENTFD 0
33 #define IVSHMEM_PEER 0
34 #define IVSHMEM_MASTER 1
36 #define IVSHMEM_REG_BAR_SIZE 0x100
38 //#define DEBUG_IVSHMEM
40 #define IVSHMEM_DPRINTF(fmt, ...) \
41 do {printf("IVSHMEM: " fmt, ## __VA_ARGS__); } while (0)
43 #define IVSHMEM_DPRINTF(fmt, ...)
51 typedef struct EventfdEntry
{
56 typedef struct IVShmemState
{
62 CharDriverState
**eventfd_chr
;
63 CharDriverState
*server_chr
;
64 MemoryRegion ivshmem_mmio
;
67 /* We might need to register the BAR before we actually have the memory.
68 * So prepare a container MemoryRegion for the BAR immediately and
69 * add a subregion when we have the memory.
73 uint64_t ivshmem_size
; /* size of shared memory region */
74 int shm_fd
; /* shared memory file descriptor */
77 int nb_peers
; /* how many guests we have space for */
78 int max_peer
; /* maximum numbered peer */
83 EventfdEntry
*eventfd_table
;
85 Error
*migration_blocker
;
90 int role_val
; /* scalar to avoid multiple string comparisons */
93 /* registers for the Inter-VM shared memory device */
94 enum ivshmem_registers
{
101 static inline uint32_t ivshmem_has_feature(IVShmemState
*ivs
,
102 unsigned int feature
) {
103 return (ivs
->features
& (1 << feature
));
106 static inline bool is_power_of_two(uint64_t x
) {
107 return (x
& (x
- 1)) == 0;
110 /* accessing registers - based on rtl8139 */
111 static void ivshmem_update_irq(IVShmemState
*s
, int val
)
114 isr
= (s
->intrstatus
& s
->intrmask
) & 0xffffffff;
116 /* don't print ISR resets */
118 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n",
119 isr
? 1 : 0, s
->intrstatus
, s
->intrmask
);
122 qemu_set_irq(s
->dev
.irq
[0], (isr
!= 0));
125 static void ivshmem_IntrMask_write(IVShmemState
*s
, uint32_t val
)
127 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val
);
131 ivshmem_update_irq(s
, val
);
134 static uint32_t ivshmem_IntrMask_read(IVShmemState
*s
)
136 uint32_t ret
= s
->intrmask
;
138 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret
);
143 static void ivshmem_IntrStatus_write(IVShmemState
*s
, uint32_t val
)
145 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val
);
149 ivshmem_update_irq(s
, val
);
153 static uint32_t ivshmem_IntrStatus_read(IVShmemState
*s
)
155 uint32_t ret
= s
->intrstatus
;
157 /* reading ISR clears all interrupts */
160 ivshmem_update_irq(s
, 0);
165 static void ivshmem_io_write(void *opaque
, target_phys_addr_t addr
,
166 uint64_t val
, unsigned size
)
168 IVShmemState
*s
= opaque
;
170 uint64_t write_one
= 1;
171 uint16_t dest
= val
>> 16;
172 uint16_t vector
= val
& 0xff;
176 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx
"\n", addr
);
180 ivshmem_IntrMask_write(s
, val
);
184 ivshmem_IntrStatus_write(s
, val
);
188 /* check that dest VM ID is reasonable */
189 if (dest
> s
->max_peer
) {
190 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest
);
194 /* check doorbell range */
195 if (vector
< s
->peers
[dest
].nb_eventfds
) {
196 IVSHMEM_DPRINTF("Writing %" PRId64
" to VM %d on vector %d\n",
197 write_one
, dest
, vector
);
198 if (write(s
->peers
[dest
].eventfds
[vector
],
199 &(write_one
), 8) != 8) {
200 IVSHMEM_DPRINTF("error writing to eventfd\n");
205 IVSHMEM_DPRINTF("Invalid VM Doorbell VM %d\n", dest
);
209 static uint64_t ivshmem_io_read(void *opaque
, target_phys_addr_t addr
,
213 IVShmemState
*s
= opaque
;
219 ret
= ivshmem_IntrMask_read(s
);
223 ret
= ivshmem_IntrStatus_read(s
);
227 /* return my VM ID if the memory is mapped */
236 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx
"\n", addr
);
243 static const MemoryRegionOps ivshmem_mmio_ops
= {
244 .read
= ivshmem_io_read
,
245 .write
= ivshmem_io_write
,
246 .endianness
= DEVICE_NATIVE_ENDIAN
,
248 .min_access_size
= 4,
249 .max_access_size
= 4,
253 static void ivshmem_receive(void *opaque
, const uint8_t *buf
, int size
)
255 IVShmemState
*s
= opaque
;
257 ivshmem_IntrStatus_write(s
, *buf
);
259 IVSHMEM_DPRINTF("ivshmem_receive 0x%02x\n", *buf
);
262 static int ivshmem_can_receive(void * opaque
)
267 static void ivshmem_event(void *opaque
, int event
)
269 IVSHMEM_DPRINTF("ivshmem_event %d\n", event
);
272 static void fake_irqfd(void *opaque
, const uint8_t *buf
, int size
) {
274 EventfdEntry
*entry
= opaque
;
275 PCIDevice
*pdev
= entry
->pdev
;
277 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev
, entry
->vector
);
278 msix_notify(pdev
, entry
->vector
);
281 static CharDriverState
* create_eventfd_chr_device(void * opaque
, int eventfd
,
284 /* create a event character device based on the passed eventfd */
285 IVShmemState
*s
= opaque
;
286 CharDriverState
* chr
;
288 chr
= qemu_chr_open_eventfd(eventfd
);
291 fprintf(stderr
, "creating eventfd for eventfd %d failed\n", eventfd
);
295 /* if MSI is supported we need multiple interrupts */
296 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
297 s
->eventfd_table
[vector
].pdev
= &s
->dev
;
298 s
->eventfd_table
[vector
].vector
= vector
;
300 qemu_chr_add_handlers(chr
, ivshmem_can_receive
, fake_irqfd
,
301 ivshmem_event
, &s
->eventfd_table
[vector
]);
303 qemu_chr_add_handlers(chr
, ivshmem_can_receive
, ivshmem_receive
,
311 static int check_shm_size(IVShmemState
*s
, int fd
) {
312 /* check that the guest isn't going to try and map more memory than the
313 * the object has allocated return -1 to indicate error */
319 if (s
->ivshmem_size
> buf
.st_size
) {
321 "IVSHMEM ERROR: Requested memory size greater"
322 " than shared object size (%" PRIu64
" > %" PRIu64
")\n",
323 s
->ivshmem_size
, (uint64_t)buf
.st_size
);
330 /* create the shared memory BAR when we are not using the server, so we can
331 * create the BAR and map the memory immediately */
332 static void create_shared_memory_BAR(IVShmemState
*s
, int fd
) {
338 ptr
= mmap(0, s
->ivshmem_size
, PROT_READ
|PROT_WRITE
, MAP_SHARED
, fd
, 0);
340 memory_region_init_ram_ptr(&s
->ivshmem
, "ivshmem.bar2",
341 s
->ivshmem_size
, ptr
);
342 vmstate_register_ram(&s
->ivshmem
, &s
->dev
.qdev
);
343 memory_region_add_subregion(&s
->bar
, 0, &s
->ivshmem
);
345 /* region for shared memory */
346 pci_register_bar(&s
->dev
, 2, PCI_BASE_ADDRESS_SPACE_MEMORY
, &s
->bar
);
349 static void close_guest_eventfds(IVShmemState
*s
, int posn
)
351 int i
, guest_curr_max
;
353 guest_curr_max
= s
->peers
[posn
].nb_eventfds
;
355 for (i
= 0; i
< guest_curr_max
; i
++) {
356 kvm_set_ioeventfd_mmio(s
->peers
[posn
].eventfds
[i
],
357 s
->mmio_addr
+ DOORBELL
, (posn
<< 16) | i
, 0, 4);
358 close(s
->peers
[posn
].eventfds
[i
]);
361 g_free(s
->peers
[posn
].eventfds
);
362 s
->peers
[posn
].nb_eventfds
= 0;
365 static void setup_ioeventfds(IVShmemState
*s
) {
369 for (i
= 0; i
<= s
->max_peer
; i
++) {
370 for (j
= 0; j
< s
->peers
[i
].nb_eventfds
; j
++) {
371 memory_region_add_eventfd(&s
->ivshmem_mmio
,
376 s
->peers
[i
].eventfds
[j
]);
381 /* this function increase the dynamic storage need to store data about other
383 static void increase_dynamic_storage(IVShmemState
*s
, int new_min_size
) {
387 old_nb_alloc
= s
->nb_peers
;
389 while (new_min_size
>= s
->nb_peers
)
390 s
->nb_peers
= s
->nb_peers
* 2;
392 IVSHMEM_DPRINTF("bumping storage to %d guests\n", s
->nb_peers
);
393 s
->peers
= g_realloc(s
->peers
, s
->nb_peers
* sizeof(Peer
));
395 /* zero out new pointers */
396 for (j
= old_nb_alloc
; j
< s
->nb_peers
; j
++) {
397 s
->peers
[j
].eventfds
= NULL
;
398 s
->peers
[j
].nb_eventfds
= 0;
402 static void ivshmem_read(void *opaque
, const uint8_t * buf
, int flags
)
404 IVShmemState
*s
= opaque
;
405 int incoming_fd
, tmp_fd
;
406 int guest_max_eventfd
;
409 memcpy(&incoming_posn
, buf
, sizeof(long));
410 /* pick off s->server_chr->msgfd and store it, posn should accompany msg */
411 tmp_fd
= qemu_chr_fe_get_msgfd(s
->server_chr
);
412 IVSHMEM_DPRINTF("posn is %ld, fd is %d\n", incoming_posn
, tmp_fd
);
414 /* make sure we have enough space for this guest */
415 if (incoming_posn
>= s
->nb_peers
) {
416 increase_dynamic_storage(s
, incoming_posn
);
420 /* if posn is positive and unseen before then this is our posn*/
421 if ((incoming_posn
>= 0) &&
422 (s
->peers
[incoming_posn
].eventfds
== NULL
)) {
423 /* receive our posn */
424 s
->vm_id
= incoming_posn
;
427 /* otherwise an fd == -1 means an existing guest has gone away */
428 IVSHMEM_DPRINTF("posn %ld has gone away\n", incoming_posn
);
429 close_guest_eventfds(s
, incoming_posn
);
434 /* because of the implementation of get_msgfd, we need a dup */
435 incoming_fd
= dup(tmp_fd
);
437 if (incoming_fd
== -1) {
438 fprintf(stderr
, "could not allocate file descriptor %s\n",
443 /* if the position is -1, then it's shared memory region fd */
444 if (incoming_posn
== -1) {
450 if (check_shm_size(s
, incoming_fd
) == -1) {
454 /* mmap the region and map into the BAR2 */
455 map_ptr
= mmap(0, s
->ivshmem_size
, PROT_READ
|PROT_WRITE
, MAP_SHARED
,
457 memory_region_init_ram_ptr(&s
->ivshmem
,
458 "ivshmem.bar2", s
->ivshmem_size
, map_ptr
);
459 vmstate_register_ram(&s
->ivshmem
, &s
->dev
.qdev
);
461 IVSHMEM_DPRINTF("guest h/w addr = %" PRIu64
", size = %" PRIu64
"\n",
462 s
->ivshmem_offset
, s
->ivshmem_size
);
464 memory_region_add_subregion(&s
->bar
, 0, &s
->ivshmem
);
466 /* only store the fd if it is successfully mapped */
467 s
->shm_fd
= incoming_fd
;
472 /* each guest has an array of eventfds, and we keep track of how many
473 * guests for each VM */
474 guest_max_eventfd
= s
->peers
[incoming_posn
].nb_eventfds
;
476 if (guest_max_eventfd
== 0) {
477 /* one eventfd per MSI vector */
478 s
->peers
[incoming_posn
].eventfds
= (int *) g_malloc(s
->vectors
*
482 /* this is an eventfd for a particular guest VM */
483 IVSHMEM_DPRINTF("eventfds[%ld][%d] = %d\n", incoming_posn
,
484 guest_max_eventfd
, incoming_fd
);
485 s
->peers
[incoming_posn
].eventfds
[guest_max_eventfd
] = incoming_fd
;
487 /* increment count for particular guest */
488 s
->peers
[incoming_posn
].nb_eventfds
++;
490 /* keep track of the maximum VM ID */
491 if (incoming_posn
> s
->max_peer
) {
492 s
->max_peer
= incoming_posn
;
495 if (incoming_posn
== s
->vm_id
) {
496 s
->eventfd_chr
[guest_max_eventfd
] = create_eventfd_chr_device(s
,
497 s
->peers
[s
->vm_id
].eventfds
[guest_max_eventfd
],
501 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
)) {
502 if (kvm_set_ioeventfd_mmio(incoming_fd
, s
->mmio_addr
+ DOORBELL
,
503 (incoming_posn
<< 16) | guest_max_eventfd
, 1, 4) < 0) {
504 fprintf(stderr
, "ivshmem: ioeventfd not available\n");
511 /* Select the MSI-X vectors used by device.
512 * ivshmem maps events to vectors statically, so
513 * we just enable all vectors on init and after reset. */
514 static void ivshmem_use_msix(IVShmemState
* s
)
518 if (!msix_present(&s
->dev
)) {
522 for (i
= 0; i
< s
->vectors
; i
++) {
523 msix_vector_use(&s
->dev
, i
);
527 static void ivshmem_reset(DeviceState
*d
)
529 IVShmemState
*s
= DO_UPCAST(IVShmemState
, dev
.qdev
, d
);
536 static uint64_t ivshmem_get_size(IVShmemState
* s
) {
541 value
= strtoull(s
->sizearg
, &ptr
, 10);
543 case 0: case 'M': case 'm':
550 fprintf(stderr
, "qemu: invalid ram size: %s\n", s
->sizearg
);
554 /* BARs must be a power of 2 */
555 if (!is_power_of_two(value
)) {
556 fprintf(stderr
, "ivshmem: size must be power of 2\n");
563 static void ivshmem_setup_msi(IVShmemState
* s
)
565 if (msix_init_exclusive_bar(&s
->dev
, s
->vectors
, 1)) {
566 IVSHMEM_DPRINTF("msix initialization failed\n");
570 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s
->vectors
);
572 /* allocate QEMU char devices for receiving interrupts */
573 s
->eventfd_table
= g_malloc0(s
->vectors
* sizeof(EventfdEntry
));
578 static void ivshmem_save(QEMUFile
* f
, void *opaque
)
580 IVShmemState
*proxy
= opaque
;
582 IVSHMEM_DPRINTF("ivshmem_save\n");
583 pci_device_save(&proxy
->dev
, f
);
585 if (ivshmem_has_feature(proxy
, IVSHMEM_MSI
)) {
586 msix_save(&proxy
->dev
, f
);
588 qemu_put_be32(f
, proxy
->intrstatus
);
589 qemu_put_be32(f
, proxy
->intrmask
);
594 static int ivshmem_load(QEMUFile
* f
, void *opaque
, int version_id
)
596 IVSHMEM_DPRINTF("ivshmem_load\n");
598 IVShmemState
*proxy
= opaque
;
601 if (version_id
> 0) {
605 if (proxy
->role_val
== IVSHMEM_PEER
) {
606 fprintf(stderr
, "ivshmem: 'peer' devices are not migratable\n");
610 ret
= pci_device_load(&proxy
->dev
, f
);
615 if (ivshmem_has_feature(proxy
, IVSHMEM_MSI
)) {
616 msix_load(&proxy
->dev
, f
);
617 ivshmem_use_msix(proxy
);
619 proxy
->intrstatus
= qemu_get_be32(f
);
620 proxy
->intrmask
= qemu_get_be32(f
);
626 static void ivshmem_write_config(PCIDevice
*pci_dev
, uint32_t address
,
627 uint32_t val
, int len
)
629 pci_default_write_config(pci_dev
, address
, val
, len
);
630 msix_write_config(pci_dev
, address
, val
, len
);
633 static int pci_ivshmem_init(PCIDevice
*dev
)
635 IVShmemState
*s
= DO_UPCAST(IVShmemState
, dev
, dev
);
638 if (s
->sizearg
== NULL
)
639 s
->ivshmem_size
= 4 << 20; /* 4 MB default */
641 s
->ivshmem_size
= ivshmem_get_size(s
);
644 register_savevm(&s
->dev
.qdev
, "ivshmem", 0, 0, ivshmem_save
, ivshmem_load
,
647 /* IRQFD requires MSI */
648 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
) &&
649 !ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
650 fprintf(stderr
, "ivshmem: ioeventfd/irqfd requires MSI\n");
654 /* check that role is reasonable */
656 if (strncmp(s
->role
, "peer", 5) == 0) {
657 s
->role_val
= IVSHMEM_PEER
;
658 } else if (strncmp(s
->role
, "master", 7) == 0) {
659 s
->role_val
= IVSHMEM_MASTER
;
661 fprintf(stderr
, "ivshmem: 'role' must be 'peer' or 'master'\n");
665 s
->role_val
= IVSHMEM_MASTER
; /* default */
668 if (s
->role_val
== IVSHMEM_PEER
) {
669 error_set(&s
->migration_blocker
, QERR_DEVICE_FEATURE_BLOCKS_MIGRATION
, "ivshmem", "peer mode");
670 migrate_add_blocker(s
->migration_blocker
);
673 pci_conf
= s
->dev
.config
;
674 pci_conf
[PCI_COMMAND
] = PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
;
676 pci_config_set_interrupt_pin(pci_conf
, 1);
680 memory_region_init_io(&s
->ivshmem_mmio
, &ivshmem_mmio_ops
, s
,
681 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE
);
683 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
)) {
687 /* region for registers*/
688 pci_register_bar(&s
->dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
,
691 memory_region_init(&s
->bar
, "ivshmem-bar2-container", s
->ivshmem_size
);
693 if ((s
->server_chr
!= NULL
) &&
694 (strncmp(s
->server_chr
->filename
, "unix:", 5) == 0)) {
695 /* if we get a UNIX socket as the parameter we will talk
696 * to the ivshmem server to receive the memory region */
698 if (s
->shmobj
!= NULL
) {
699 fprintf(stderr
, "WARNING: do not specify both 'chardev' "
700 "and 'shm' with ivshmem\n");
703 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
704 s
->server_chr
->filename
);
706 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
707 ivshmem_setup_msi(s
);
710 /* we allocate enough space for 16 guests and grow as needed */
714 /* allocate/initialize space for interrupt handling */
715 s
->peers
= g_malloc0(s
->nb_peers
* sizeof(Peer
));
717 pci_register_bar(&s
->dev
, 2,
718 PCI_BASE_ADDRESS_SPACE_MEMORY
, &s
->bar
);
720 s
->eventfd_chr
= g_malloc0(s
->vectors
* sizeof(CharDriverState
*));
722 qemu_chr_add_handlers(s
->server_chr
, ivshmem_can_receive
, ivshmem_read
,
725 /* just map the file immediately, we're not using a server */
728 if (s
->shmobj
== NULL
) {
729 fprintf(stderr
, "Must specify 'chardev' or 'shm' to ivshmem\n");
732 IVSHMEM_DPRINTF("using shm_open (shm object = %s)\n", s
->shmobj
);
734 /* try opening with O_EXCL and if it succeeds zero the memory
735 * by truncating to 0 */
736 if ((fd
= shm_open(s
->shmobj
, O_CREAT
|O_RDWR
|O_EXCL
,
737 S_IRWXU
|S_IRWXG
|S_IRWXO
)) > 0) {
738 /* truncate file to length PCI device's memory */
739 if (ftruncate(fd
, s
->ivshmem_size
) != 0) {
740 fprintf(stderr
, "ivshmem: could not truncate shared file\n");
743 } else if ((fd
= shm_open(s
->shmobj
, O_CREAT
|O_RDWR
,
744 S_IRWXU
|S_IRWXG
|S_IRWXO
)) < 0) {
745 fprintf(stderr
, "ivshmem: could not open shared file\n");
750 if (check_shm_size(s
, fd
) == -1) {
754 create_shared_memory_BAR(s
, fd
);
758 s
->dev
.config_write
= ivshmem_write_config
;
763 static int pci_ivshmem_uninit(PCIDevice
*dev
)
765 IVShmemState
*s
= DO_UPCAST(IVShmemState
, dev
, dev
);
767 if (s
->migration_blocker
) {
768 migrate_del_blocker(s
->migration_blocker
);
769 error_free(s
->migration_blocker
);
772 memory_region_destroy(&s
->ivshmem_mmio
);
773 memory_region_del_subregion(&s
->bar
, &s
->ivshmem
);
774 vmstate_unregister_ram(&s
->ivshmem
, &s
->dev
.qdev
);
775 memory_region_destroy(&s
->ivshmem
);
776 memory_region_destroy(&s
->bar
);
777 unregister_savevm(&dev
->qdev
, "ivshmem", s
);
782 static Property ivshmem_properties
[] = {
783 DEFINE_PROP_CHR("chardev", IVShmemState
, server_chr
),
784 DEFINE_PROP_STRING("size", IVShmemState
, sizearg
),
785 DEFINE_PROP_UINT32("vectors", IVShmemState
, vectors
, 1),
786 DEFINE_PROP_BIT("ioeventfd", IVShmemState
, features
, IVSHMEM_IOEVENTFD
, false),
787 DEFINE_PROP_BIT("msi", IVShmemState
, features
, IVSHMEM_MSI
, true),
788 DEFINE_PROP_STRING("shm", IVShmemState
, shmobj
),
789 DEFINE_PROP_STRING("role", IVShmemState
, role
),
790 DEFINE_PROP_END_OF_LIST(),
793 static void ivshmem_class_init(ObjectClass
*klass
, void *data
)
795 DeviceClass
*dc
= DEVICE_CLASS(klass
);
796 PCIDeviceClass
*k
= PCI_DEVICE_CLASS(klass
);
798 k
->init
= pci_ivshmem_init
;
799 k
->exit
= pci_ivshmem_uninit
;
800 k
->vendor_id
= PCI_VENDOR_ID_REDHAT_QUMRANET
;
801 k
->device_id
= 0x1110;
802 k
->class_id
= PCI_CLASS_MEMORY_RAM
;
803 dc
->reset
= ivshmem_reset
;
804 dc
->props
= ivshmem_properties
;
807 static TypeInfo ivshmem_info
= {
809 .parent
= TYPE_PCI_DEVICE
,
810 .instance_size
= sizeof(IVShmemState
),
811 .class_init
= ivshmem_class_init
,
814 static void ivshmem_register_types(void)
816 type_register_static(&ivshmem_info
);
819 type_init(ivshmem_register_types
)