virt/kvm/arm/vgic/vgic-mmio.c

   1 /*
   2  * VGIC MMIO handling functions
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  */
  13
  14 #include <linux/bitops.h>
  15 #include <linux/bsearch.h>
  16 #include <linux/kvm.h>
  17 #include <linux/kvm_host.h>
  18 #include <kvm/iodev.h>
  19 #include <kvm/arm_vgic.h>
  20
  21 #include "vgic.h"
  22 #include "vgic-mmio.h"
  23
  24 unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu,
  25                                  gpa_t addr, unsigned int len)
  26 {
  27         return 0;
  28 }
  29
  30 unsigned long vgic_mmio_read_rao(struct kvm_vcpu *vcpu,
  31                                  gpa_t addr, unsigned int len)
  32 {
  33         return -1UL;
  34 }
  35
  36 void vgic_mmio_write_wi(struct kvm_vcpu *vcpu, gpa_t addr,
  37                         unsigned int len, unsigned long val)
  38 {
  39         /* Ignore */
  40 }
  41
  42 /*
  43  * Read accesses to both GICD_ICENABLER and GICD_ISENABLER return the value
  44  * of the enabled bit, so there is only one function for both here.
  45  */
  46 unsigned long vgic_mmio_read_enable(struct kvm_vcpu *vcpu,
  47                                     gpa_t addr, unsigned int len)
  48 {
  49         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
  50         u32 value = 0;
  51         int i;
  52
  53         /* Loop over all IRQs affected by this read */
  54         for (i = 0; i < len * 8; i++) {
  55                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
  56
  57                 if (irq->enabled)
  58                         value |= (1U << i);
  59
  60                 vgic_put_irq(vcpu->kvm, irq);
  61         }
  62
  63         return value;
  64 }
  65
  66 void vgic_mmio_write_senable(struct kvm_vcpu *vcpu,
  67                              gpa_t addr, unsigned int len,
  68                              unsigned long val)
  69 {
  70         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
  71         int i;
  72
  73         for_each_set_bit(i, &val, len * 8) {
  74                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
  75
  76                 spin_lock(&irq->irq_lock);
  77                 irq->enabled = true;
  78                 vgic_queue_irq_unlock(vcpu->kvm, irq);
  79
  80                 vgic_put_irq(vcpu->kvm, irq);
  81         }
  82 }
  83
  84 void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu,
  85                              gpa_t addr, unsigned int len,
  86                              unsigned long val)
  87 {
  88         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
  89         int i;
  90
  91         for_each_set_bit(i, &val, len * 8) {
  92                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
  93
  94                 spin_lock(&irq->irq_lock);
  95
  96                 irq->enabled = false;
  97
  98                 spin_unlock(&irq->irq_lock);
  99                 vgic_put_irq(vcpu->kvm, irq);
 100         }
 101 }
 102
 103 unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
 104                                      gpa_t addr, unsigned int len)
 105 {
 106         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 107         u32 value = 0;
 108         int i;
 109
 110         /* Loop over all IRQs affected by this read */
 111         for (i = 0; i < len * 8; i++) {
 112                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 113
 114                 if (irq_is_pending(irq))
 115                         value |= (1U << i);
 116
 117                 vgic_put_irq(vcpu->kvm, irq);
 118         }
 119
 120         return value;
 121 }
 122
 123 void vgic_mmio_write_spending(struct kvm_vcpu *vcpu,
 124                               gpa_t addr, unsigned int len,
 125                               unsigned long val)
 126 {
 127         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 128         int i;
 129
 130         for_each_set_bit(i, &val, len * 8) {
 131                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 132
 133                 spin_lock(&irq->irq_lock);
 134                 irq->pending_latch = true;
 135
 136                 vgic_queue_irq_unlock(vcpu->kvm, irq);
 137                 vgic_put_irq(vcpu->kvm, irq);
 138         }
 139 }
 140
 141 void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu,
 142                               gpa_t addr, unsigned int len,
 143                               unsigned long val)
 144 {
 145         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 146         int i;
 147
 148         for_each_set_bit(i, &val, len * 8) {
 149                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 150
 151                 spin_lock(&irq->irq_lock);
 152
 153                 irq->pending_latch = false;
 154
 155                 spin_unlock(&irq->irq_lock);
 156                 vgic_put_irq(vcpu->kvm, irq);
 157         }
 158 }
 159
 160 unsigned long vgic_mmio_read_active(struct kvm_vcpu *vcpu,
 161                                     gpa_t addr, unsigned int len)
 162 {
 163         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 164         u32 value = 0;
 165         int i;
 166
 167         /* Loop over all IRQs affected by this read */
 168         for (i = 0; i < len * 8; i++) {
 169                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 170
 171                 if (irq->active)
 172                         value |= (1U << i);
 173
 174                 vgic_put_irq(vcpu->kvm, irq);
 175         }
 176
 177         return value;
 178 }
 179
 180 static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
 181                                     bool new_active_state)
 182 {
 183         spin_lock(&irq->irq_lock);
 184         /*
 185          * If this virtual IRQ was written into a list register, we
 186          * have to make sure the CPU that runs the VCPU thread has
 187          * synced back LR state to the struct vgic_irq.  We can only
 188          * know this for sure, when either this irq is not assigned to
 189          * anyone's AP list anymore, or the VCPU thread is not
 190          * running on any CPUs.
 191          *
 192          * In the opposite case, we know the VCPU thread may be on its
 193          * way back from the guest and still has to sync back this
 194          * IRQ, so we release and re-acquire the spin_lock to let the
 195          * other thread sync back the IRQ.
 196          */
 197         while (irq->vcpu && /* IRQ may have state in an LR somewhere */
 198                irq->vcpu->cpu != -1) /* VCPU thread is running */
 199                 cond_resched_lock(&irq->irq_lock);
 200
 201         irq->active = new_active_state;
 202         if (new_active_state)
 203                 vgic_queue_irq_unlock(vcpu->kvm, irq);
 204         else
 205                 spin_unlock(&irq->irq_lock);
 206 }
 207
 208 /*
 209  * If we are fiddling with an IRQ's active state, we have to make sure the IRQ
 210  * is not queued on some running VCPU's LRs, because then the change to the
 211  * active state can be overwritten when the VCPU's state is synced coming back
 212  * from the guest.
 213  *
 214  * For shared interrupts, we have to stop all the VCPUs because interrupts can
 215  * be migrated while we don't hold the IRQ locks and we don't want to be
 216  * chasing moving targets.
 217  *
 218  * For private interrupts, we only have to make sure the single and only VCPU
 219  * that can potentially queue the IRQ is stopped.
 220  */
 221 static void vgic_change_active_prepare(struct kvm_vcpu *vcpu, u32 intid)
 222 {
 223         if (intid < VGIC_NR_PRIVATE_IRQS)
 224                 kvm_arm_halt_vcpu(vcpu);
 225         else
 226                 kvm_arm_halt_guest(vcpu->kvm);
 227 }
 228
 229 /* See vgic_change_active_prepare */
 230 static void vgic_change_active_finish(struct kvm_vcpu *vcpu, u32 intid)
 231 {
 232         if (intid < VGIC_NR_PRIVATE_IRQS)
 233                 kvm_arm_resume_vcpu(vcpu);
 234         else
 235                 kvm_arm_resume_guest(vcpu->kvm);
 236 }
 237
 238 void vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
 239                              gpa_t addr, unsigned int len,
 240                              unsigned long val)
 241 {
 242         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 243         int i;
 244
 245         vgic_change_active_prepare(vcpu, intid);
 246         for_each_set_bit(i, &val, len * 8) {
 247                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 248                 vgic_mmio_change_active(vcpu, irq, false);
 249                 vgic_put_irq(vcpu->kvm, irq);
 250         }
 251         vgic_change_active_finish(vcpu, intid);
 252 }
 253
 254 void vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
 255                              gpa_t addr, unsigned int len,
 256                              unsigned long val)
 257 {
 258         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 259         int i;
 260
 261         vgic_change_active_prepare(vcpu, intid);
 262         for_each_set_bit(i, &val, len * 8) {
 263                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 264                 vgic_mmio_change_active(vcpu, irq, true);
 265                 vgic_put_irq(vcpu->kvm, irq);
 266         }
 267         vgic_change_active_finish(vcpu, intid);
 268 }
 269
 270 unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu,
 271                                       gpa_t addr, unsigned int len)
 272 {
 273         u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
 274         int i;
 275         u64 val = 0;
 276
 277         for (i = 0; i < len; i++) {
 278                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 279
 280                 val |= (u64)irq->priority << (i * 8);
 281
 282                 vgic_put_irq(vcpu->kvm, irq);
 283         }
 284
 285         return val;
 286 }
 287
 288 /*
 289  * We currently don't handle changing the priority of an interrupt that
 290  * is already pending on a VCPU. If there is a need for this, we would
 291  * need to make this VCPU exit and re-evaluate the priorities, potentially
 292  * leading to this interrupt getting presented now to the guest (if it has
 293  * been masked by the priority mask before).
 294  */
 295 void vgic_mmio_write_priority(struct kvm_vcpu *vcpu,
 296                               gpa_t addr, unsigned int len,
 297                               unsigned long val)
 298 {
 299         u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
 300         int i;
 301
 302         for (i = 0; i < len; i++) {
 303                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 304
 305                 spin_lock(&irq->irq_lock);
 306                 /* Narrow the priority range to what we actually support */
 307                 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);
 308                 spin_unlock(&irq->irq_lock);
 309
 310                 vgic_put_irq(vcpu->kvm, irq);
 311         }
 312 }
 313
 314 unsigned long vgic_mmio_read_config(struct kvm_vcpu *vcpu,
 315                                     gpa_t addr, unsigned int len)
 316 {
 317         u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
 318         u32 value = 0;
 319         int i;
 320
 321         for (i = 0; i < len * 4; i++) {
 322                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 323
 324                 if (irq->config == VGIC_CONFIG_EDGE)
 325                         value |= (2U << (i * 2));
 326
 327                 vgic_put_irq(vcpu->kvm, irq);
 328         }
 329
 330         return value;
 331 }
 332
 333 void vgic_mmio_write_config(struct kvm_vcpu *vcpu,
 334                             gpa_t addr, unsigned int len,
 335                             unsigned long val)
 336 {
 337         u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
 338         int i;
 339
 340         for (i = 0; i < len * 4; i++) {
 341                 struct vgic_irq *irq;
 342
 343                 /*
 344                  * The configuration cannot be changed for SGIs in general,
 345                  * for PPIs this is IMPLEMENTATION DEFINED. The arch timer
 346                  * code relies on PPIs being level triggered, so we also
 347                  * make them read-only here.
 348                  */
 349                 if (intid + i < VGIC_NR_PRIVATE_IRQS)
 350                         continue;
 351
 352                 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 353                 spin_lock(&irq->irq_lock);
 354
 355                 if (test_bit(i * 2 + 1, &val))
 356                         irq->config = VGIC_CONFIG_EDGE;
 357                 else
 358                         irq->config = VGIC_CONFIG_LEVEL;
 359
 360                 spin_unlock(&irq->irq_lock);
 361                 vgic_put_irq(vcpu->kvm, irq);
 362         }
 363 }
 364
 365 static int match_region(const void *key, const void *elt)
 366 {
 367         const unsigned int offset = (unsigned long)key;
 368         const struct vgic_register_region *region = elt;
 369
 370         if (offset < region->reg_offset)
 371                 return -1;
 372
 373         if (offset >= region->reg_offset + region->len)
 374                 return 1;
 375
 376         return 0;
 377 }
 378
 379 /* Find the proper register handler entry given a certain address offset. */
 380 static const struct vgic_register_region *
 381 vgic_find_mmio_region(const struct vgic_register_region *region, int nr_regions,
 382                       unsigned int offset)
 383 {
 384         return bsearch((void *)(uintptr_t)offset, region, nr_regions,
 385                        sizeof(region[0]), match_region);
 386 }
 387
 388 void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 389 {
 390         if (kvm_vgic_global_state.type == VGIC_V2)
 391                 vgic_v2_set_vmcr(vcpu, vmcr);
 392         else
 393                 vgic_v3_set_vmcr(vcpu, vmcr);
 394 }
 395
 396 void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 397 {
 398         if (kvm_vgic_global_state.type == VGIC_V2)
 399                 vgic_v2_get_vmcr(vcpu, vmcr);
 400         else
 401                 vgic_v3_get_vmcr(vcpu, vmcr);
 402 }
 403
 404 /*
 405  * kvm_mmio_read_buf() returns a value in a format where it can be converted
 406  * to a byte array and be directly observed as the guest wanted it to appear
 407  * in memory if it had done the store itself, which is LE for the GIC, as the
 408  * guest knows the GIC is always LE.
 409  *
 410  * We convert this value to the CPUs native format to deal with it as a data
 411  * value.
 412  */
 413 unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len)
 414 {
 415         unsigned long data = kvm_mmio_read_buf(val, len);
 416
 417         switch (len) {
 418         case 1:
 419                 return data;
 420         case 2:
 421                 return le16_to_cpu(data);
 422         case 4:
 423                 return le32_to_cpu(data);
 424         default:
 425                 return le64_to_cpu(data);
 426         }
 427 }
 428
 429 /*
 430  * kvm_mmio_write_buf() expects a value in a format such that if converted to
 431  * a byte array it is observed as the guest would see it if it could perform
 432  * the load directly.  Since the GIC is LE, and the guest knows this, the
 433  * guest expects a value in little endian format.
 434  *
 435  * We convert the data value from the CPUs native format to LE so that the
 436  * value is returned in the proper format.
 437  */
 438 void vgic_data_host_to_mmio_bus(void *buf, unsigned int len,
 439                                 unsigned long data)
 440 {
 441         switch (len) {
 442         case 1:
 443                 break;
 444         case 2:
 445                 data = cpu_to_le16(data);
 446                 break;
 447         case 4:
 448                 data = cpu_to_le32(data);
 449                 break;
 450         default:
 451                 data = cpu_to_le64(data);
 452         }
 453
 454         kvm_mmio_write_buf(buf, len, data);
 455 }
 456
 457 static
 458 struct vgic_io_device *kvm_to_vgic_iodev(const struct kvm_io_device *dev)
 459 {
 460         return container_of(dev, struct vgic_io_device, dev);
 461 }
 462
 463 static bool check_region(const struct kvm *kvm,
 464                          const struct vgic_register_region *region,
 465                          gpa_t addr, int len)
 466 {
 467         int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
 468
 469         switch (len) {
 470         case sizeof(u8):
 471                 flags = VGIC_ACCESS_8bit;
 472                 break;
 473         case sizeof(u32):
 474                 flags = VGIC_ACCESS_32bit;
 475                 break;
 476         case sizeof(u64):
 477                 flags = VGIC_ACCESS_64bit;
 478                 break;
 479         default:
 480                 return false;
 481         }
 482
 483         if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) {
 484                 if (!region->bits_per_irq)
 485                         return true;
 486
 487                 /* Do we access a non-allocated IRQ? */
 488                 return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs;
 489         }
 490
 491         return false;
 492 }
 493
 494 const struct vgic_register_region *
 495 vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
 496                      gpa_t addr, int len)
 497 {
 498         const struct vgic_register_region *region;
 499
 500         region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
 501                                        addr - iodev->base_addr);
 502         if (!region || !check_region(vcpu->kvm, region, addr, len))
 503                 return NULL;
 504
 505         return region;
 506 }
 507
 508 static int vgic_uaccess_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 509                              gpa_t addr, u32 *val)
 510 {
 511         struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
 512         const struct vgic_register_region *region;
 513         struct kvm_vcpu *r_vcpu;
 514
 515         region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32));
 516         if (!region) {
 517                 *val = 0;
 518                 return 0;
 519         }
 520
 521         r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;
 522         if (region->uaccess_read)
 523                 *val = region->uaccess_read(r_vcpu, addr, sizeof(u32));
 524         else
 525                 *val = region->read(r_vcpu, addr, sizeof(u32));
 526
 527         return 0;
 528 }
 529
 530 static int vgic_uaccess_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 531                               gpa_t addr, const u32 *val)
 532 {
 533         struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
 534         const struct vgic_register_region *region;
 535         struct kvm_vcpu *r_vcpu;
 536
 537         region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32));
 538         if (!region)
 539                 return 0;
 540
 541         r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;
 542         if (region->uaccess_write)
 543                 region->uaccess_write(r_vcpu, addr, sizeof(u32), *val);
 544         else
 545                 region->write(r_vcpu, addr, sizeof(u32), *val);
 546
 547         return 0;
 548 }
 549
 550 /*
 551  * Userland access to VGIC registers.
 552  */
 553 int vgic_uaccess(struct kvm_vcpu *vcpu, struct vgic_io_device *dev,
 554                  bool is_write, int offset, u32 *val)
 555 {
 556         if (is_write)
 557                 return vgic_uaccess_write(vcpu, &dev->dev, offset, val);
 558         else
 559                 return vgic_uaccess_read(vcpu, &dev->dev, offset, val);
 560 }
 561
 562 static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 563                               gpa_t addr, int len, void *val)
 564 {
 565         struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
 566         const struct vgic_register_region *region;
 567         unsigned long data = 0;
 568
 569         region = vgic_get_mmio_region(vcpu, iodev, addr, len);
 570         if (!region) {
 571                 memset(val, 0, len);
 572                 return 0;
 573         }
 574
 575         switch (iodev->iodev_type) {
 576         case IODEV_CPUIF:
 577                 data = region->read(vcpu, addr, len);
 578                 break;
 579         case IODEV_DIST:
 580                 data = region->read(vcpu, addr, len);
 581                 break;
 582         case IODEV_REDIST:
 583                 data = region->read(iodev->redist_vcpu, addr, len);
 584                 break;
 585         case IODEV_ITS:
 586                 data = region->its_read(vcpu->kvm, iodev->its, addr, len);
 587                 break;
 588         }
 589
 590         vgic_data_host_to_mmio_bus(val, len, data);
 591         return 0;
 592 }
 593
 594 static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 595                                gpa_t addr, int len, const void *val)
 596 {
 597         struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
 598         const struct vgic_register_region *region;
 599         unsigned long data = vgic_data_mmio_bus_to_host(val, len);
 600
 601         region = vgic_get_mmio_region(vcpu, iodev, addr, len);
 602         if (!region)
 603                 return 0;
 604
 605         switch (iodev->iodev_type) {
 606         case IODEV_CPUIF:
 607                 region->write(vcpu, addr, len, data);
 608                 break;
 609         case IODEV_DIST:
 610                 region->write(vcpu, addr, len, data);
 611                 break;
 612         case IODEV_REDIST:
 613                 region->write(iodev->redist_vcpu, addr, len, data);
 614                 break;
 615         case IODEV_ITS:
 616                 region->its_write(vcpu->kvm, iodev->its, addr, len, data);
 617                 break;
 618         }
 619
 620         return 0;
 621 }
 622
 623 struct kvm_io_device_ops kvm_io_gic_ops = {
 624         .read = dispatch_mmio_read,
 625         .write = dispatch_mmio_write,
 626 };
 627
 628 int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
 629                              enum vgic_type type)
 630 {
 631         struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev;
 632         int ret = 0;
 633         unsigned int len;
 634
 635         switch (type) {
 636         case VGIC_V2:
 637                 len = vgic_v2_init_dist_iodev(io_device);
 638                 break;
 639         case VGIC_V3:
 640                 len = vgic_v3_init_dist_iodev(io_device);
 641                 break;
 642         default:
 643                 BUG_ON(1);
 644         }
 645
 646         io_device->base_addr = dist_base_address;
 647         io_device->iodev_type = IODEV_DIST;
 648         io_device->redist_vcpu = NULL;
 649
 650         mutex_lock(&kvm->slots_lock);
 651         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist_base_address,
 652                                       len, &io_device->dev);
 653         mutex_unlock(&kvm->slots_lock);
 654
 655         return ret;
 656 }