drivers/irqchip/irq-gic-v4.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/interrupt.h>
   8 #include <linux/irq.h>
   9 #include <linux/irqdomain.h>
  10 #include <linux/msi.h>
  11 #include <linux/sched.h>
  12
  13 #include <linux/irqchip/arm-gic-v4.h>
  14
  15 /*
  16  * WARNING: The blurb below assumes that you understand the
  17  * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
  18  * translated into GICv4 commands. So it effectively targets at most
  19  * two individuals. You know who you are.
  20  *
  21  * The core GICv4 code is designed to *avoid* exposing too much of the
  22  * core GIC code (that would in turn leak into the hypervisor code),
  23  * and instead provide a hypervisor agnostic interface to the HW (of
  24  * course, the astute reader will quickly realize that hypervisor
  25  * agnostic actually means KVM-specific - what were you thinking?).
  26  *
  27  * In order to achieve a modicum of isolation, we try to hide most of
  28  * the GICv4 "stuff" behind normal irqchip operations:
  29  *
  30  * - Any guest-visible VLPI is backed by a Linux interrupt (and a
  31  *   physical LPI which gets unmapped when the guest maps the
  32  *   VLPI). This allows the same DevID/EventID pair to be either
  33  *   mapped to the LPI (host) or the VLPI (guest). Note that this is
  34  *   exclusive, and you cannot have both.
  35  *
  36  * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
  37  *
  38  * - Guest INT/CLEAR commands are implemented through
  39  *   irq_set_irqchip_state().
  40  *
  41  * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
  42  *   issuing an INV after changing a priority) gets shoved into the
  43  *   irq_set_vcpu_affinity() method. While this is quite horrible
  44  *   (let's face it, this is the irqchip version of an ioctl), it
  45  *   confines the crap to a single location. And map/unmap really is
  46  *   about setting the affinity of a VLPI to a vcpu, so only INV is
  47  *   majorly out of place. So there.
  48  *
  49  * A number of commands are simply not provided by this interface, as
  50  * they do not make direct sense. For example, MAPD is purely local to
  51  * the virtual ITS (because it references a virtual device, and the
  52  * physical ITS is still very much in charge of the physical
  53  * device). Same goes for things like MAPC (the physical ITS deals
  54  * with the actual vPE affinity, and not the braindead concept of
  55  * collection). SYNC is not provided either, as each and every command
  56  * is followed by a VSYNC. This could be relaxed in the future, should
  57  * this be seen as a bottleneck (yes, this means *never*).
  58  *
  59  * But handling VLPIs is only one side of the job of the GICv4
  60  * code. The other (darker) side is to take care of the doorbell
  61  * interrupts which are delivered when a VLPI targeting a non-running
  62  * vcpu is being made pending.
  63  *
  64  * The choice made here is that each vcpu (VPE in old northern GICv4
  65  * dialect) gets a single doorbell LPI, no matter how many interrupts
  66  * are targeting it. This has a nice property, which is that the
  67  * interrupt becomes a handle for the VPE, and that the hypervisor
  68  * code can manipulate it through the normal interrupt API:
  69  *
  70  * - VMs (or rather the VM abstraction that matters to the GIC)
  71  *   contain an irq domain where each interrupt maps to a VPE. In
  72  *   turn, this domain sits on top of the normal LPI allocator, and a
  73  *   specially crafted irq_chip implementation.
  74  *
  75  * - mask/unmask do what is expected on the doorbell interrupt.
  76  *
  77  * - irq_set_affinity is used to move a VPE from one redistributor to
  78  *   another.
  79  *
  80  * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
  81  *   creating a new sub-API, namely scheduling/descheduling a VPE
  82  *   (which involves programming GICR_V{PROP,PEND}BASER) and
  83  *   performing INVALL operations.
  84  */
  85
  86 static struct irq_domain *gic_domain;
  87 static const struct irq_domain_ops *vpe_domain_ops;
  88 static const struct irq_domain_ops *sgi_domain_ops;
  89
  90 #ifdef CONFIG_ARM64
  91 #include <asm/cpufeature.h>
  92
  93 bool gic_cpuif_has_vsgi(void)
  94 {
  95         unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
  96
  97         fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_GIC_SHIFT);
  98
  99         return fld >= 0x3;
 100 }
 101 #else
 102 bool gic_cpuif_has_vsgi(void)
 103 {
 104         return false;
 105 }
 106 #endif
 107
 108 static bool has_v4_1(void)
 109 {
 110         return !!sgi_domain_ops;
 111 }
 112
 113 static bool has_v4_1_sgi(void)
 114 {
 115         return has_v4_1() && gic_cpuif_has_vsgi();
 116 }
 117
 118 static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
 119 {
 120         char *name;
 121         int sgi_base;
 122
 123         if (!has_v4_1_sgi())
 124                 return 0;
 125
 126         name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current));
 127         if (!name)
 128                 goto err;
 129
 130         vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx);
 131         if (!vpe->fwnode)
 132                 goto err;
 133
 134         kfree(name);
 135         name = NULL;
 136
 137         vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16,
 138                                                    sgi_domain_ops, vpe);
 139         if (!vpe->sgi_domain)
 140                 goto err;
 141
 142         sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16,
 143                                                NUMA_NO_NODE, vpe,
 144                                                false, NULL);
 145         if (sgi_base <= 0)
 146                 goto err;
 147
 148         return 0;
 149
 150 err:
 151         if (vpe->sgi_domain)
 152                 irq_domain_remove(vpe->sgi_domain);
 153         if (vpe->fwnode)
 154                 irq_domain_free_fwnode(vpe->fwnode);
 155         kfree(name);
 156         return -ENOMEM;
 157 }
 158
 159 int its_alloc_vcpu_irqs(struct its_vm *vm)
 160 {
 161         int vpe_base_irq, i;
 162
 163         vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
 164                                                       task_pid_nr(current));
 165         if (!vm->fwnode)
 166                 goto err;
 167
 168         vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
 169                                                  vm->fwnode, vpe_domain_ops,
 170                                                  vm);
 171         if (!vm->domain)
 172                 goto err;
 173
 174         for (i = 0; i < vm->nr_vpes; i++) {
 175                 vm->vpes[i]->its_vm = vm;
 176                 vm->vpes[i]->idai = true;
 177         }
 178
 179         vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
 180                                                NUMA_NO_NODE, vm,
 181                                                false, NULL);
 182         if (vpe_base_irq <= 0)
 183                 goto err;
 184
 185         for (i = 0; i < vm->nr_vpes; i++) {
 186                 int ret;
 187                 vm->vpes[i]->irq = vpe_base_irq + i;
 188                 ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
 189                 if (ret)
 190                         goto err;
 191         }
 192
 193         return 0;
 194
 195 err:
 196         if (vm->domain)
 197                 irq_domain_remove(vm->domain);
 198         if (vm->fwnode)
 199                 irq_domain_free_fwnode(vm->fwnode);
 200
 201         return -ENOMEM;
 202 }
 203
 204 static void its_free_sgi_irqs(struct its_vm *vm)
 205 {
 206         int i;
 207
 208         if (!has_v4_1_sgi())
 209                 return;
 210
 211         for (i = 0; i < vm->nr_vpes; i++) {
 212                 unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
 213
 214                 if (WARN_ON(!irq))
 215                         continue;
 216
 217                 irq_domain_free_irqs(irq, 16);
 218                 irq_domain_remove(vm->vpes[i]->sgi_domain);
 219                 irq_domain_free_fwnode(vm->vpes[i]->fwnode);
 220         }
 221 }
 222
 223 void its_free_vcpu_irqs(struct its_vm *vm)
 224 {
 225         its_free_sgi_irqs(vm);
 226         irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
 227         irq_domain_remove(vm->domain);
 228         irq_domain_free_fwnode(vm->fwnode);
 229 }
 230
 231 static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
 232 {
 233         return irq_set_vcpu_affinity(vpe->irq, info);
 234 }
 235
 236 int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
 237 {
 238         struct irq_desc *desc = irq_to_desc(vpe->irq);
 239         struct its_cmd_info info = { };
 240         int ret;
 241
 242         WARN_ON(preemptible());
 243
 244         info.cmd_type = DESCHEDULE_VPE;
 245         if (has_v4_1()) {
 246                 /* GICv4.1 can directly deal with doorbells */
 247                 info.req_db = db;
 248         } else {
 249                 /* Undo the nested disable_irq() calls... */
 250                 while (db && irqd_irq_disabled(&desc->irq_data))
 251                         enable_irq(vpe->irq);
 252         }
 253
 254         ret = its_send_vpe_cmd(vpe, &info);
 255         if (!ret)
 256                 vpe->resident = false;
 257
 258         vpe->ready = false;
 259
 260         return ret;
 261 }
 262
 263 int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
 264 {
 265         struct its_cmd_info info = { };
 266         int ret;
 267
 268         WARN_ON(preemptible());
 269
 270         info.cmd_type = SCHEDULE_VPE;
 271         if (has_v4_1()) {
 272                 info.g0en = g0en;
 273                 info.g1en = g1en;
 274         } else {
 275                 /* Disabled the doorbell, as we're about to enter the guest */
 276                 disable_irq_nosync(vpe->irq);
 277         }
 278
 279         ret = its_send_vpe_cmd(vpe, &info);
 280         if (!ret)
 281                 vpe->resident = true;
 282
 283         return ret;
 284 }
 285
 286 int its_commit_vpe(struct its_vpe *vpe)
 287 {
 288         struct its_cmd_info info = {
 289                 .cmd_type = COMMIT_VPE,
 290         };
 291         int ret;
 292
 293         WARN_ON(preemptible());
 294
 295         ret = its_send_vpe_cmd(vpe, &info);
 296         if (!ret)
 297                 vpe->ready = true;
 298
 299         return ret;
 300 }
 301
 302
 303 int its_invall_vpe(struct its_vpe *vpe)
 304 {
 305         struct its_cmd_info info = {
 306                 .cmd_type = INVALL_VPE,
 307         };
 308
 309         return its_send_vpe_cmd(vpe, &info);
 310 }
 311
 312 int its_map_vlpi(int irq, struct its_vlpi_map *map)
 313 {
 314         struct its_cmd_info info = {
 315                 .cmd_type = MAP_VLPI,
 316                 {
 317                         .map      = map,
 318                 },
 319         };
 320         int ret;
 321
 322         /*
 323          * The host will never see that interrupt firing again, so it
 324          * is vital that we don't do any lazy masking.
 325          */
 326         irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
 327
 328         ret = irq_set_vcpu_affinity(irq, &info);
 329         if (ret)
 330                 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
 331
 332         return ret;
 333 }
 334
 335 int its_get_vlpi(int irq, struct its_vlpi_map *map)
 336 {
 337         struct its_cmd_info info = {
 338                 .cmd_type = GET_VLPI,
 339                 {
 340                         .map      = map,
 341                 },
 342         };
 343
 344         return irq_set_vcpu_affinity(irq, &info);
 345 }
 346
 347 int its_unmap_vlpi(int irq)
 348 {
 349         irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
 350         return irq_set_vcpu_affinity(irq, NULL);
 351 }
 352
 353 int its_prop_update_vlpi(int irq, u8 config, bool inv)
 354 {
 355         struct its_cmd_info info = {
 356                 .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
 357                 {
 358                         .config   = config,
 359                 },
 360         };
 361
 362         return irq_set_vcpu_affinity(irq, &info);
 363 }
 364
 365 int its_prop_update_vsgi(int irq, u8 priority, bool group)
 366 {
 367         struct its_cmd_info info = {
 368                 .cmd_type = PROP_UPDATE_VSGI,
 369                 {
 370                         .priority       = priority,
 371                         .group          = group,
 372                 },
 373         };
 374
 375         return irq_set_vcpu_affinity(irq, &info);
 376 }
 377
 378 int its_init_v4(struct irq_domain *domain,
 379                 const struct irq_domain_ops *vpe_ops,
 380                 const struct irq_domain_ops *sgi_ops)
 381 {
 382         if (domain) {
 383                 pr_info("ITS: Enabling GICv4 support\n");
 384                 gic_domain = domain;
 385                 vpe_domain_ops = vpe_ops;
 386                 sgi_domain_ops = sgi_ops;
 387                 return 0;
 388         }
 389
 390         pr_err("ITS: No GICv4 VPE domain allocated\n");
 391         return -ENODEV;
 392 }