virt/kvm/arm/vgic/vgic-v4.c

   1 /*
   2  * Copyright (C) 2017 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/interrupt.h>
  19 #include <linux/irq.h>
  20 #include <linux/irqdomain.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/irqchip/arm-gic-v3.h>
  23
  24 #include "vgic.h"
  25
  26 /*
  27  * How KVM uses GICv4 (insert rude comments here):
  28  *
  29  * The vgic-v4 layer acts as a bridge between several entities:
  30  * - The GICv4 ITS representation offered by the ITS driver
  31  * - VFIO, which is in charge of the PCI endpoint
  32  * - The virtual ITS, which is the only thing the guest sees
  33  *
  34  * The configuration of VLPIs is triggered by a callback from VFIO,
  35  * instructing KVM that a PCI device has been configured to deliver
  36  * MSIs to a vITS.
  37  *
  38  * kvm_vgic_v4_set_forwarding() is thus called with the routing entry,
  39  * and this is used to find the corresponding vITS data structures
  40  * (ITS instance, device, event and irq) using a process that is
  41  * extremely similar to the injection of an MSI.
  42  *
  43  * At this stage, we can link the guest's view of an LPI (uniquely
  44  * identified by the routing entry) and the host irq, using the GICv4
  45  * driver mapping operation. Should the mapping succeed, we've then
  46  * successfully upgraded the guest's LPI to a VLPI. We can then start
  47  * with updating GICv4's view of the property table and generating an
  48  * INValidation in order to kickstart the delivery of this VLPI to the
  49  * guest directly, without software intervention. Well, almost.
  50  *
  51  * When the PCI endpoint is deconfigured, this operation is reversed
  52  * with VFIO calling kvm_vgic_v4_unset_forwarding().
  53  *
  54  * Once the VLPI has been mapped, it needs to follow any change the
  55  * guest performs on its LPI through the vITS. For that, a number of
  56  * command handlers have hooks to communicate these changes to the HW:
  57  * - Any invalidation triggers a call to its_prop_update_vlpi()
  58  * - The INT command results in a irq_set_irqchip_state(), which
  59  *   generates an INT on the corresponding VLPI.
  60  * - The CLEAR command results in a irq_set_irqchip_state(), which
  61  *   generates an CLEAR on the corresponding VLPI.
  62  * - DISCARD translates into an unmap, similar to a call to
  63  *   kvm_vgic_v4_unset_forwarding().
  64  * - MOVI is translated by an update of the existing mapping, changing
  65  *   the target vcpu, resulting in a VMOVI being generated.
  66  * - MOVALL is translated by a string of mapping updates (similar to
  67  *   the handling of MOVI). MOVALL is horrible.
  68  *
  69  * Note that a DISCARD/MAPTI sequence emitted from the guest without
  70  * reprogramming the PCI endpoint after MAPTI does not result in a
  71  * VLPI being mapped, as there is no callback from VFIO (the guest
  72  * will get the interrupt via the normal SW injection). Fixing this is
  73  * not trivial, and requires some horrible messing with the VFIO
  74  * internals. Not fun. Don't do that.
  75  *
  76  * Then there is the scheduling. Each time a vcpu is about to run on a
  77  * physical CPU, KVM must tell the corresponding redistributor about
  78  * it. And if we've migrated our vcpu from one CPU to another, we must
  79  * tell the ITS (so that the messages reach the right redistributor).
  80  * This is done in two steps: first issue a irq_set_affinity() on the
  81  * irq corresponding to the vcpu, then call its_schedule_vpe(). You
  82  * must be in a non-preemptible context. On exit, another call to
  83  * its_schedule_vpe() tells the redistributor that we're done with the
  84  * vcpu.
  85  *
  86  * Finally, the doorbell handling: Each vcpu is allocated an interrupt
  87  * which will fire each time a VLPI is made pending whilst the vcpu is
  88  * not running. Each time the vcpu gets blocked, the doorbell
  89  * interrupt gets enabled. When the vcpu is unblocked (for whatever
  90  * reason), the doorbell interrupt is disabled.
  91  */
  92
  93 #define DB_IRQ_FLAGS    (IRQ_NOAUTOEN | IRQ_DISABLE_UNLAZY | IRQ_NO_BALANCING)
  94
  95 static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info)
  96 {
  97         struct kvm_vcpu *vcpu = info;
  98
  99         vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last = true;
 100         kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
 101         kvm_vcpu_kick(vcpu);
 102
 103         return IRQ_HANDLED;
 104 }
 105
 106 /**
 107  * vgic_v4_init - Initialize the GICv4 data structures
 108  * @kvm:        Pointer to the VM being initialized
 109  *
 110  * We may be called each time a vITS is created, or when the
 111  * vgic is initialized. This relies on kvm->lock to be
 112  * held. In both cases, the number of vcpus should now be
 113  * fixed.
 114  */
 115 int vgic_v4_init(struct kvm *kvm)
 116 {
 117         struct vgic_dist *dist = &kvm->arch.vgic;
 118         struct kvm_vcpu *vcpu;
 119         int i, nr_vcpus, ret;
 120
 121         if (!vgic_supports_direct_msis(kvm))
 122                 return 0; /* Nothing to see here... move along. */
 123
 124         if (dist->its_vm.vpes)
 125                 return 0;
 126
 127         nr_vcpus = atomic_read(&kvm->online_vcpus);
 128
 129         dist->its_vm.vpes = kzalloc(sizeof(*dist->its_vm.vpes) * nr_vcpus,
 130                                     GFP_KERNEL);
 131         if (!dist->its_vm.vpes)
 132                 return -ENOMEM;
 133
 134         dist->its_vm.nr_vpes = nr_vcpus;
 135
 136         kvm_for_each_vcpu(i, vcpu, kvm)
 137                 dist->its_vm.vpes[i] = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
 138
 139         ret = its_alloc_vcpu_irqs(&dist->its_vm);
 140         if (ret < 0) {
 141                 kvm_err("VPE IRQ allocation failure\n");
 142                 kfree(dist->its_vm.vpes);
 143                 dist->its_vm.nr_vpes = 0;
 144                 dist->its_vm.vpes = NULL;
 145                 return ret;
 146         }
 147
 148         kvm_for_each_vcpu(i, vcpu, kvm) {
 149                 int irq = dist->its_vm.vpes[i]->irq;
 150
 151                 /*
 152                  * Don't automatically enable the doorbell, as we're
 153                  * flipping it back and forth when the vcpu gets
 154                  * blocked. Also disable the lazy disabling, as the
 155                  * doorbell could kick us out of the guest too
 156                  * early...
 157                  */
 158                 irq_set_status_flags(irq, DB_IRQ_FLAGS);
 159                 ret = request_irq(irq, vgic_v4_doorbell_handler,
 160                                   0, "vcpu", vcpu);
 161                 if (ret) {
 162                         kvm_err("failed to allocate vcpu IRQ%d\n", irq);
 163                         /*
 164                          * Trick: adjust the number of vpes so we know
 165                          * how many to nuke on teardown...
 166                          */
 167                         dist->its_vm.nr_vpes = i;
 168                         break;
 169                 }
 170         }
 171
 172         if (ret)
 173                 vgic_v4_teardown(kvm);
 174
 175         return ret;
 176 }
 177
 178 /**
 179  * vgic_v4_teardown - Free the GICv4 data structures
 180  * @kvm:        Pointer to the VM being destroyed
 181  *
 182  * Relies on kvm->lock to be held.
 183  */
 184 void vgic_v4_teardown(struct kvm *kvm)
 185 {
 186         struct its_vm *its_vm = &kvm->arch.vgic.its_vm;
 187         int i;
 188
 189         if (!its_vm->vpes)
 190                 return;
 191
 192         for (i = 0; i < its_vm->nr_vpes; i++) {
 193                 struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, i);
 194                 int irq = its_vm->vpes[i]->irq;
 195
 196                 irq_clear_status_flags(irq, DB_IRQ_FLAGS);
 197                 free_irq(irq, vcpu);
 198         }
 199
 200         its_free_vcpu_irqs(its_vm);
 201         kfree(its_vm->vpes);
 202         its_vm->nr_vpes = 0;
 203         its_vm->vpes = NULL;
 204 }
 205
 206 int vgic_v4_sync_hwstate(struct kvm_vcpu *vcpu)
 207 {
 208         if (!vgic_supports_direct_msis(vcpu->kvm))
 209                 return 0;
 210
 211         return its_schedule_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe, false);
 212 }
 213
 214 int vgic_v4_flush_hwstate(struct kvm_vcpu *vcpu)
 215 {
 216         int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq;
 217         int err;
 218
 219         if (!vgic_supports_direct_msis(vcpu->kvm))
 220                 return 0;
 221
 222         /*
 223          * Before making the VPE resident, make sure the redistributor
 224          * corresponding to our current CPU expects us here. See the
 225          * doc in drivers/irqchip/irq-gic-v4.c to understand how this
 226          * turns into a VMOVP command at the ITS level.
 227          */
 228         err = irq_set_affinity(irq, cpumask_of(smp_processor_id()));
 229         if (err)
 230                 return err;
 231
 232         err = its_schedule_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe, true);
 233         if (err)
 234                 return err;
 235
 236         /*
 237          * Now that the VPE is resident, let's get rid of a potential
 238          * doorbell interrupt that would still be pending.
 239          */
 240         err = irq_set_irqchip_state(irq, IRQCHIP_STATE_PENDING, false);
 241
 242         return err;
 243 }
 244
 245 static struct vgic_its *vgic_get_its(struct kvm *kvm,
 246                                      struct kvm_kernel_irq_routing_entry *irq_entry)
 247 {
 248         struct kvm_msi msi  = (struct kvm_msi) {
 249                 .address_lo     = irq_entry->msi.address_lo,
 250                 .address_hi     = irq_entry->msi.address_hi,
 251                 .data           = irq_entry->msi.data,
 252                 .flags          = irq_entry->msi.flags,
 253                 .devid          = irq_entry->msi.devid,
 254         };
 255
 256         return vgic_msi_to_its(kvm, &msi);
 257 }
 258
 259 int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq,
 260                                struct kvm_kernel_irq_routing_entry *irq_entry)
 261 {
 262         struct vgic_its *its;
 263         struct vgic_irq *irq;
 264         struct its_vlpi_map map;
 265         int ret;
 266
 267         if (!vgic_supports_direct_msis(kvm))
 268                 return 0;
 269
 270         /*
 271          * Get the ITS, and escape early on error (not a valid
 272          * doorbell for any of our vITSs).
 273          */
 274         its = vgic_get_its(kvm, irq_entry);
 275         if (IS_ERR(its))
 276                 return 0;
 277
 278         mutex_lock(&its->its_lock);
 279
 280         /* Perform then actual DevID/EventID -> LPI translation. */
 281         ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid,
 282                                    irq_entry->msi.data, &irq);
 283         if (ret)
 284                 goto out;
 285
 286         /*
 287          * Emit the mapping request. If it fails, the ITS probably
 288          * isn't v4 compatible, so let's silently bail out. Holding
 289          * the ITS lock should ensure that nothing can modify the
 290          * target vcpu.
 291          */
 292         map = (struct its_vlpi_map) {
 293                 .vm             = &kvm->arch.vgic.its_vm,
 294                 .vpe            = &irq->target_vcpu->arch.vgic_cpu.vgic_v3.its_vpe,
 295                 .vintid         = irq->intid,
 296                 .properties     = ((irq->priority & 0xfc) |
 297                                    (irq->enabled ? LPI_PROP_ENABLED : 0) |
 298                                    LPI_PROP_GROUP1),
 299                 .db_enabled     = true,
 300         };
 301
 302         ret = its_map_vlpi(virq, &map);
 303         if (ret)
 304                 goto out;
 305
 306         irq->hw         = true;
 307         irq->host_irq   = virq;
 308
 309 out:
 310         mutex_unlock(&its->its_lock);
 311         return ret;
 312 }
 313
 314 int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int virq,
 315                                  struct kvm_kernel_irq_routing_entry *irq_entry)
 316 {
 317         struct vgic_its *its;
 318         struct vgic_irq *irq;
 319         int ret;
 320
 321         if (!vgic_supports_direct_msis(kvm))
 322                 return 0;
 323
 324         /*
 325          * Get the ITS, and escape early on error (not a valid
 326          * doorbell for any of our vITSs).
 327          */
 328         its = vgic_get_its(kvm, irq_entry);
 329         if (IS_ERR(its))
 330                 return 0;
 331
 332         mutex_lock(&its->its_lock);
 333
 334         ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid,
 335                                    irq_entry->msi.data, &irq);
 336         if (ret)
 337                 goto out;
 338
 339         WARN_ON(!(irq->hw && irq->host_irq == virq));
 340         irq->hw = false;
 341         ret = its_unmap_vlpi(virq);
 342
 343 out:
 344         mutex_unlock(&its->its_lock);
 345         return ret;
 346 }
 347
 348 void kvm_vgic_v4_enable_doorbell(struct kvm_vcpu *vcpu)
 349 {
 350         if (vgic_supports_direct_msis(vcpu->kvm)) {
 351                 int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq;
 352                 if (irq)
 353                         enable_irq(irq);
 354         }
 355 }
 356
 357 void kvm_vgic_v4_disable_doorbell(struct kvm_vcpu *vcpu)
 358 {
 359         if (vgic_supports_direct_msis(vcpu->kvm)) {
 360                 int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq;
 361                 if (irq)
 362                         disable_irq(irq);
 363         }
 364 }