kernel/irq/ipi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2015 Imagination Technologies Ltd
   4  * Author: Qais Yousef <qais.yousef@imgtec.com>
   5  *
   6  * This file contains driver APIs to the IPI subsystem.
   7  */
   8
   9 #define pr_fmt(fmt) "genirq/ipi: " fmt
  10
  11 #include <linux/irqdomain.h>
  12 #include <linux/irq.h>
  13
  14 /**
  15  * irq_reserve_ipi() - Setup an IPI to destination cpumask
  16  * @domain:     IPI domain
  17  * @dest:       cpumask of cpus which can receive the IPI
  18  *
  19  * Allocate a virq that can be used to send IPI to any CPU in dest mask.
  20  *
  21  * On success it'll return linux irq number and error code on failure
  22  */
  23 int irq_reserve_ipi(struct irq_domain *domain,
  24                              const struct cpumask *dest)
  25 {
  26         unsigned int nr_irqs, offset;
  27         struct irq_data *data;
  28         int virq, i;
  29
  30         if (!domain ||!irq_domain_is_ipi(domain)) {
  31                 pr_warn("Reservation on a non IPI domain\n");
  32                 return -EINVAL;
  33         }
  34
  35         if (!cpumask_subset(dest, cpu_possible_mask)) {
  36                 pr_warn("Reservation is not in possible_cpu_mask\n");
  37                 return -EINVAL;
  38         }
  39
  40         nr_irqs = cpumask_weight(dest);
  41         if (!nr_irqs) {
  42                 pr_warn("Reservation for empty destination mask\n");
  43                 return -EINVAL;
  44         }
  45
  46         if (irq_domain_is_ipi_single(domain)) {
  47                 /*
  48                  * If the underlying implementation uses a single HW irq on
  49                  * all cpus then we only need a single Linux irq number for
  50                  * it. We have no restrictions vs. the destination mask. The
  51                  * underlying implementation can deal with holes nicely.
  52                  */
  53                 nr_irqs = 1;
  54                 offset = 0;
  55         } else {
  56                 unsigned int next;
  57
  58                 /*
  59                  * The IPI requires a separate HW irq on each CPU. We require
  60                  * that the destination mask is consecutive. If an
  61                  * implementation needs to support holes, it can reserve
  62                  * several IPI ranges.
  63                  */
  64                 offset = cpumask_first(dest);
  65                 /*
  66                  * Find a hole and if found look for another set bit after the
  67                  * hole. For now we don't support this scenario.
  68                  */
  69                 next = cpumask_next_zero(offset, dest);
  70                 if (next < nr_cpu_ids)
  71                         next = cpumask_next(next, dest);
  72                 if (next < nr_cpu_ids) {
  73                         pr_warn("Destination mask has holes\n");
  74                         return -EINVAL;
  75                 }
  76         }
  77
  78         virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
  79         if (virq <= 0) {
  80                 pr_warn("Can't reserve IPI, failed to alloc descs\n");
  81                 return -ENOMEM;
  82         }
  83
  84         virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
  85                                        (void *) dest, true, NULL);
  86
  87         if (virq <= 0) {
  88                 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
  89                 goto free_descs;
  90         }
  91
  92         for (i = 0; i < nr_irqs; i++) {
  93                 data = irq_get_irq_data(virq + i);
  94                 cpumask_copy(data->common->affinity, dest);
  95                 data->common->ipi_offset = offset;
  96                 irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
  97         }
  98         return virq;
  99
 100 free_descs:
 101         irq_free_descs(virq, nr_irqs);
 102         return -EBUSY;
 103 }
 104
 105 /**
 106  * irq_destroy_ipi() - unreserve an IPI that was previously allocated
 107  * @irq:        linux irq number to be destroyed
 108  * @dest:       cpumask of cpus which should have the IPI removed
 109  *
 110  * The IPIs allocated with irq_reserve_ipi() are retuerned to the system
 111  * destroying all virqs associated with them.
 112  *
 113  * Return 0 on success or error code on failure.
 114  */
 115 int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
 116 {
 117         struct irq_data *data = irq_get_irq_data(irq);
 118         struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
 119         struct irq_domain *domain;
 120         unsigned int nr_irqs;
 121
 122         if (!irq || !data || !ipimask)
 123                 return -EINVAL;
 124
 125         domain = data->domain;
 126         if (WARN_ON(domain == NULL))
 127                 return -EINVAL;
 128
 129         if (!irq_domain_is_ipi(domain)) {
 130                 pr_warn("Trying to destroy a non IPI domain!\n");
 131                 return -EINVAL;
 132         }
 133
 134         if (WARN_ON(!cpumask_subset(dest, ipimask)))
 135                 /*
 136                  * Must be destroying a subset of CPUs to which this IPI
 137                  * was set up to target
 138                  */
 139                 return -EINVAL;
 140
 141         if (irq_domain_is_ipi_per_cpu(domain)) {
 142                 irq = irq + cpumask_first(dest) - data->common->ipi_offset;
 143                 nr_irqs = cpumask_weight(dest);
 144         } else {
 145                 nr_irqs = 1;
 146         }
 147
 148         irq_domain_free_irqs(irq, nr_irqs);
 149         return 0;
 150 }
 151
 152 /**
 153  * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu
 154  * @irq:        linux irq number
 155  * @cpu:        the target cpu
 156  *
 157  * When dealing with coprocessors IPI, we need to inform the coprocessor of
 158  * the hwirq it needs to use to receive and send IPIs.
 159  *
 160  * Returns hwirq value on success and INVALID_HWIRQ on failure.
 161  */
 162 irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
 163 {
 164         struct irq_data *data = irq_get_irq_data(irq);
 165         struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
 166
 167         if (!data || !ipimask || cpu >= nr_cpu_ids)
 168                 return INVALID_HWIRQ;
 169
 170         if (!cpumask_test_cpu(cpu, ipimask))
 171                 return INVALID_HWIRQ;
 172
 173         /*
 174          * Get the real hardware irq number if the underlying implementation
 175          * uses a separate irq per cpu. If the underlying implementation uses
 176          * a single hardware irq for all cpus then the IPI send mechanism
 177          * needs to take care of the cpu destinations.
 178          */
 179         if (irq_domain_is_ipi_per_cpu(data->domain))
 180                 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
 181
 182         return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
 183 }
 184 EXPORT_SYMBOL_GPL(ipi_get_hwirq);
 185
 186 static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
 187                            const struct cpumask *dest, unsigned int cpu)
 188 {
 189         struct cpumask *ipimask = irq_data_get_affinity_mask(data);
 190
 191         if (!chip || !ipimask)
 192                 return -EINVAL;
 193
 194         if (!chip->ipi_send_single && !chip->ipi_send_mask)
 195                 return -EINVAL;
 196
 197         if (cpu >= nr_cpu_ids)
 198                 return -EINVAL;
 199
 200         if (dest) {
 201                 if (!cpumask_subset(dest, ipimask))
 202                         return -EINVAL;
 203         } else {
 204                 if (!cpumask_test_cpu(cpu, ipimask))
 205                         return -EINVAL;
 206         }
 207         return 0;
 208 }
 209
 210 /**
 211  * __ipi_send_single - send an IPI to a target Linux SMP CPU
 212  * @desc:       pointer to irq_desc of the IRQ
 213  * @cpu:        destination CPU, must in the destination mask passed to
 214  *              irq_reserve_ipi()
 215  *
 216  * This function is for architecture or core code to speed up IPI sending. Not
 217  * usable from driver code.
 218  *
 219  * Returns zero on success and negative error number on failure.
 220  */
 221 int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
 222 {
 223         struct irq_data *data = irq_desc_get_irq_data(desc);
 224         struct irq_chip *chip = irq_data_get_irq_chip(data);
 225
 226 #ifdef DEBUG
 227         /*
 228          * Minimise the overhead by omitting the checks for Linux SMP IPIs.
 229          * Since the callers should be arch or core code which is generally
 230          * trusted, only check for errors when debugging.
 231          */
 232         if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
 233                 return -EINVAL;
 234 #endif
 235         if (!chip->ipi_send_single) {
 236                 chip->ipi_send_mask(data, cpumask_of(cpu));
 237                 return 0;
 238         }
 239
 240         /* FIXME: Store this information in irqdata flags */
 241         if (irq_domain_is_ipi_per_cpu(data->domain) &&
 242             cpu != data->common->ipi_offset) {
 243                 /* use the correct data for that cpu */
 244                 unsigned irq = data->irq + cpu - data->common->ipi_offset;
 245
 246                 data = irq_get_irq_data(irq);
 247         }
 248         chip->ipi_send_single(data, cpu);
 249         return 0;
 250 }
 251
 252 /**
 253  * ipi_send_mask - send an IPI to target Linux SMP CPU(s)
 254  * @desc:       pointer to irq_desc of the IRQ
 255  * @dest:       dest CPU(s), must be a subset of the mask passed to
 256  *              irq_reserve_ipi()
 257  *
 258  * This function is for architecture or core code to speed up IPI sending. Not
 259  * usable from driver code.
 260  *
 261  * Returns zero on success and negative error number on failure.
 262  */
 263 int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
 264 {
 265         struct irq_data *data = irq_desc_get_irq_data(desc);
 266         struct irq_chip *chip = irq_data_get_irq_chip(data);
 267         unsigned int cpu;
 268
 269 #ifdef DEBUG
 270         /*
 271          * Minimise the overhead by omitting the checks for Linux SMP IPIs.
 272          * Since the callers should be arch or core code which is generally
 273          * trusted, only check for errors when debugging.
 274          */
 275         if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
 276                 return -EINVAL;
 277 #endif
 278         if (chip->ipi_send_mask) {
 279                 chip->ipi_send_mask(data, dest);
 280                 return 0;
 281         }
 282
 283         if (irq_domain_is_ipi_per_cpu(data->domain)) {
 284                 unsigned int base = data->irq;
 285
 286                 for_each_cpu(cpu, dest) {
 287                         unsigned irq = base + cpu - data->common->ipi_offset;
 288
 289                         data = irq_get_irq_data(irq);
 290                         chip->ipi_send_single(data, cpu);
 291                 }
 292         } else {
 293                 for_each_cpu(cpu, dest)
 294                         chip->ipi_send_single(data, cpu);
 295         }
 296         return 0;
 297 }
 298
 299 /**
 300  * ipi_send_single - Send an IPI to a single CPU
 301  * @virq:       linux irq number from irq_reserve_ipi()
 302  * @cpu:        destination CPU, must in the destination mask passed to
 303  *              irq_reserve_ipi()
 304  *
 305  * Returns zero on success and negative error number on failure.
 306  */
 307 int ipi_send_single(unsigned int virq, unsigned int cpu)
 308 {
 309         struct irq_desc *desc = irq_to_desc(virq);
 310         struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
 311         struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
 312
 313         if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
 314                 return -EINVAL;
 315
 316         return __ipi_send_single(desc, cpu);
 317 }
 318 EXPORT_SYMBOL_GPL(ipi_send_single);
 319
 320 /**
 321  * ipi_send_mask - Send an IPI to target CPU(s)
 322  * @virq:       linux irq number from irq_reserve_ipi()
 323  * @dest:       dest CPU(s), must be a subset of the mask passed to
 324  *              irq_reserve_ipi()
 325  *
 326  * Returns zero on success and negative error number on failure.
 327  */
 328 int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
 329 {
 330         struct irq_desc *desc = irq_to_desc(virq);
 331         struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
 332         struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
 333
 334         if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
 335                 return -EINVAL;
 336
 337         return __ipi_send_mask(desc, dest);
 338 }
 339 EXPORT_SYMBOL_GPL(ipi_send_mask);