hw/virtio/virtio-iommu.c

   1 /*
   2  * virtio-iommu device
   3  *
   4  * Copyright (c) 2020 Red Hat, Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2 or later, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "qemu/iov.h"
  23 #include "qemu-common.h"
  24 #include "hw/qdev-properties.h"
  25 #include "hw/virtio/virtio.h"
  26 #include "sysemu/kvm.h"
  27 #include "qapi/error.h"
  28 #include "qemu/error-report.h"
  29 #include "trace.h"
  30
  31 #include "standard-headers/linux/virtio_ids.h"
  32
  33 #include "hw/virtio/virtio-bus.h"
  34 #include "hw/virtio/virtio-access.h"
  35 #include "hw/virtio/virtio-iommu.h"
  36 #include "hw/pci/pci_bus.h"
  37 #include "hw/pci/pci.h"
  38
  39 /* Max size */
  40 #define VIOMMU_DEFAULT_QUEUE_SIZE 256
  41 #define VIOMMU_PROBE_SIZE 512
  42
  43 typedef struct VirtIOIOMMUDomain {
  44     uint32_t id;
  45     GTree *mappings;
  46     QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list;
  47 } VirtIOIOMMUDomain;
  48
  49 typedef struct VirtIOIOMMUEndpoint {
  50     uint32_t id;
  51     VirtIOIOMMUDomain *domain;
  52     QLIST_ENTRY(VirtIOIOMMUEndpoint) next;
  53 } VirtIOIOMMUEndpoint;
  54
  55 typedef struct VirtIOIOMMUInterval {
  56     uint64_t low;
  57     uint64_t high;
  58 } VirtIOIOMMUInterval;
  59
  60 typedef struct VirtIOIOMMUMapping {
  61     uint64_t phys_addr;
  62     uint32_t flags;
  63 } VirtIOIOMMUMapping;
  64
  65 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev)
  66 {
  67     return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn);
  68 }
  69
  70 /**
  71  * The bus number is used for lookup when SID based operations occur.
  72  * In that case we lazily populate the IOMMUPciBus array from the bus hash
  73  * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus
  74  * numbers may not be always initialized yet.
  75  */
  76 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num)
  77 {
  78     IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num];
  79
  80     if (!iommu_pci_bus) {
  81         GHashTableIter iter;
  82
  83         g_hash_table_iter_init(&iter, s->as_by_busptr);
  84         while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
  85             if (pci_bus_num(iommu_pci_bus->bus) == bus_num) {
  86                 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus;
  87                 return iommu_pci_bus;
  88             }
  89         }
  90         return NULL;
  91     }
  92     return iommu_pci_bus;
  93 }
  94
  95 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid)
  96 {
  97     uint8_t bus_n, devfn;
  98     IOMMUPciBus *iommu_pci_bus;
  99     IOMMUDevice *dev;
 100
 101     bus_n = PCI_BUS_NUM(sid);
 102     iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n);
 103     if (iommu_pci_bus) {
 104         devfn = sid & PCI_DEVFN_MAX;
 105         dev = iommu_pci_bus->pbdev[devfn];
 106         if (dev) {
 107             return &dev->iommu_mr;
 108         }
 109     }
 110     return NULL;
 111 }
 112
 113 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
 114 {
 115     VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a;
 116     VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b;
 117
 118     if (inta->high < intb->low) {
 119         return -1;
 120     } else if (intb->high < inta->low) {
 121         return 1;
 122     } else {
 123         return 0;
 124     }
 125 }
 126
 127 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep)
 128 {
 129     if (!ep->domain) {
 130         return;
 131     }
 132     QLIST_REMOVE(ep, next);
 133     ep->domain = NULL;
 134 }
 135
 136 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s,
 137                                                       uint32_t ep_id)
 138 {
 139     VirtIOIOMMUEndpoint *ep;
 140
 141     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
 142     if (ep) {
 143         return ep;
 144     }
 145     if (!virtio_iommu_mr(s, ep_id)) {
 146         return NULL;
 147     }
 148     ep = g_malloc0(sizeof(*ep));
 149     ep->id = ep_id;
 150     trace_virtio_iommu_get_endpoint(ep_id);
 151     g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep);
 152     return ep;
 153 }
 154
 155 static void virtio_iommu_put_endpoint(gpointer data)
 156 {
 157     VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data;
 158
 159     if (ep->domain) {
 160         virtio_iommu_detach_endpoint_from_domain(ep);
 161     }
 162
 163     trace_virtio_iommu_put_endpoint(ep->id);
 164     g_free(ep);
 165 }
 166
 167 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s,
 168                                                   uint32_t domain_id)
 169 {
 170     VirtIOIOMMUDomain *domain;
 171
 172     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
 173     if (domain) {
 174         return domain;
 175     }
 176     domain = g_malloc0(sizeof(*domain));
 177     domain->id = domain_id;
 178     domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
 179                                    NULL, (GDestroyNotify)g_free,
 180                                    (GDestroyNotify)g_free);
 181     g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain);
 182     QLIST_INIT(&domain->endpoint_list);
 183     trace_virtio_iommu_get_domain(domain_id);
 184     return domain;
 185 }
 186
 187 static void virtio_iommu_put_domain(gpointer data)
 188 {
 189     VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data;
 190     VirtIOIOMMUEndpoint *iter, *tmp;
 191
 192     QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) {
 193         virtio_iommu_detach_endpoint_from_domain(iter);
 194     }
 195     g_tree_destroy(domain->mappings);
 196     trace_virtio_iommu_put_domain(domain->id);
 197     g_free(domain);
 198 }
 199
 200 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque,
 201                                               int devfn)
 202 {
 203     VirtIOIOMMU *s = opaque;
 204     IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus);
 205     static uint32_t mr_index;
 206     IOMMUDevice *sdev;
 207
 208     if (!sbus) {
 209         sbus = g_malloc0(sizeof(IOMMUPciBus) +
 210                          sizeof(IOMMUDevice *) * PCI_DEVFN_MAX);
 211         sbus->bus = bus;
 212         g_hash_table_insert(s->as_by_busptr, bus, sbus);
 213     }
 214
 215     sdev = sbus->pbdev[devfn];
 216     if (!sdev) {
 217         char *name = g_strdup_printf("%s-%d-%d",
 218                                      TYPE_VIRTIO_IOMMU_MEMORY_REGION,
 219                                      mr_index++, devfn);
 220         sdev = sbus->pbdev[devfn] = g_malloc0(sizeof(IOMMUDevice));
 221
 222         sdev->viommu = s;
 223         sdev->bus = bus;
 224         sdev->devfn = devfn;
 225
 226         trace_virtio_iommu_init_iommu_mr(name);
 227
 228         memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr),
 229                                  TYPE_VIRTIO_IOMMU_MEMORY_REGION,
 230                                  OBJECT(s), name,
 231                                  UINT64_MAX);
 232         address_space_init(&sdev->as,
 233                            MEMORY_REGION(&sdev->iommu_mr), TYPE_VIRTIO_IOMMU);
 234         g_free(name);
 235     }
 236     return &sdev->as;
 237 }
 238
 239 static int virtio_iommu_attach(VirtIOIOMMU *s,
 240                                struct virtio_iommu_req_attach *req)
 241 {
 242     uint32_t domain_id = le32_to_cpu(req->domain);
 243     uint32_t ep_id = le32_to_cpu(req->endpoint);
 244     VirtIOIOMMUDomain *domain;
 245     VirtIOIOMMUEndpoint *ep;
 246
 247     trace_virtio_iommu_attach(domain_id, ep_id);
 248
 249     ep = virtio_iommu_get_endpoint(s, ep_id);
 250     if (!ep) {
 251         return VIRTIO_IOMMU_S_NOENT;
 252     }
 253
 254     if (ep->domain) {
 255         VirtIOIOMMUDomain *previous_domain = ep->domain;
 256         /*
 257          * the device is already attached to a domain,
 258          * detach it first
 259          */
 260         virtio_iommu_detach_endpoint_from_domain(ep);
 261         if (QLIST_EMPTY(&previous_domain->endpoint_list)) {
 262             g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id));
 263         }
 264     }
 265
 266     domain = virtio_iommu_get_domain(s, domain_id);
 267     QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next);
 268
 269     ep->domain = domain;
 270
 271     return VIRTIO_IOMMU_S_OK;
 272 }
 273
 274 static int virtio_iommu_detach(VirtIOIOMMU *s,
 275                                struct virtio_iommu_req_detach *req)
 276 {
 277     uint32_t domain_id = le32_to_cpu(req->domain);
 278     uint32_t ep_id = le32_to_cpu(req->endpoint);
 279     VirtIOIOMMUDomain *domain;
 280     VirtIOIOMMUEndpoint *ep;
 281
 282     trace_virtio_iommu_detach(domain_id, ep_id);
 283
 284     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
 285     if (!ep) {
 286         return VIRTIO_IOMMU_S_NOENT;
 287     }
 288
 289     domain = ep->domain;
 290
 291     if (!domain || domain->id != domain_id) {
 292         return VIRTIO_IOMMU_S_INVAL;
 293     }
 294
 295     virtio_iommu_detach_endpoint_from_domain(ep);
 296
 297     if (QLIST_EMPTY(&domain->endpoint_list)) {
 298         g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id));
 299     }
 300     return VIRTIO_IOMMU_S_OK;
 301 }
 302
 303 static int virtio_iommu_map(VirtIOIOMMU *s,
 304                             struct virtio_iommu_req_map *req)
 305 {
 306     uint32_t domain_id = le32_to_cpu(req->domain);
 307     uint64_t phys_start = le64_to_cpu(req->phys_start);
 308     uint64_t virt_start = le64_to_cpu(req->virt_start);
 309     uint64_t virt_end = le64_to_cpu(req->virt_end);
 310     uint32_t flags = le32_to_cpu(req->flags);
 311     VirtIOIOMMUDomain *domain;
 312     VirtIOIOMMUInterval *interval;
 313     VirtIOIOMMUMapping *mapping;
 314
 315     if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) {
 316         return VIRTIO_IOMMU_S_INVAL;
 317     }
 318
 319     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
 320     if (!domain) {
 321         return VIRTIO_IOMMU_S_NOENT;
 322     }
 323
 324     interval = g_malloc0(sizeof(*interval));
 325
 326     interval->low = virt_start;
 327     interval->high = virt_end;
 328
 329     mapping = g_tree_lookup(domain->mappings, (gpointer)interval);
 330     if (mapping) {
 331         g_free(interval);
 332         return VIRTIO_IOMMU_S_INVAL;
 333     }
 334
 335     trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags);
 336
 337     mapping = g_malloc0(sizeof(*mapping));
 338     mapping->phys_addr = phys_start;
 339     mapping->flags = flags;
 340
 341     g_tree_insert(domain->mappings, interval, mapping);
 342
 343     return VIRTIO_IOMMU_S_OK;
 344 }
 345
 346 static int virtio_iommu_unmap(VirtIOIOMMU *s,
 347                               struct virtio_iommu_req_unmap *req)
 348 {
 349     uint32_t domain_id = le32_to_cpu(req->domain);
 350     uint64_t virt_start = le64_to_cpu(req->virt_start);
 351     uint64_t virt_end = le64_to_cpu(req->virt_end);
 352     VirtIOIOMMUMapping *iter_val;
 353     VirtIOIOMMUInterval interval, *iter_key;
 354     VirtIOIOMMUDomain *domain;
 355     int ret = VIRTIO_IOMMU_S_OK;
 356
 357     trace_virtio_iommu_unmap(domain_id, virt_start, virt_end);
 358
 359     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
 360     if (!domain) {
 361         return VIRTIO_IOMMU_S_NOENT;
 362     }
 363     interval.low = virt_start;
 364     interval.high = virt_end;
 365
 366     while (g_tree_lookup_extended(domain->mappings, &interval,
 367                                   (void **)&iter_key, (void**)&iter_val)) {
 368         uint64_t current_low = iter_key->low;
 369         uint64_t current_high = iter_key->high;
 370
 371         if (interval.low <= current_low && interval.high >= current_high) {
 372             g_tree_remove(domain->mappings, iter_key);
 373             trace_virtio_iommu_unmap_done(domain_id, current_low, current_high);
 374         } else {
 375             ret = VIRTIO_IOMMU_S_RANGE;
 376             break;
 377         }
 378     }
 379     return ret;
 380 }
 381
 382 static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep,
 383                                                uint8_t *buf, size_t free)
 384 {
 385     struct virtio_iommu_probe_resv_mem prop = {};
 386     size_t size = sizeof(prop), length = size - sizeof(prop.head), total;
 387     int i;
 388
 389     total = size * s->nb_reserved_regions;
 390
 391     if (total > free) {
 392         return -ENOSPC;
 393     }
 394
 395     for (i = 0; i < s->nb_reserved_regions; i++) {
 396         unsigned subtype = s->reserved_regions[i].type;
 397
 398         assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED ||
 399                subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI);
 400         prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM);
 401         prop.head.length = cpu_to_le16(length);
 402         prop.subtype = subtype;
 403         prop.start = cpu_to_le64(s->reserved_regions[i].low);
 404         prop.end = cpu_to_le64(s->reserved_regions[i].high);
 405
 406         memcpy(buf, &prop, size);
 407
 408         trace_virtio_iommu_fill_resv_property(ep, prop.subtype,
 409                                               prop.start, prop.end);
 410         buf += size;
 411     }
 412     return total;
 413 }
 414
 415 /**
 416  * virtio_iommu_probe - Fill the probe request buffer with
 417  * the properties the device is able to return
 418  */
 419 static int virtio_iommu_probe(VirtIOIOMMU *s,
 420                               struct virtio_iommu_req_probe *req,
 421                               uint8_t *buf)
 422 {
 423     uint32_t ep_id = le32_to_cpu(req->endpoint);
 424     size_t free = VIOMMU_PROBE_SIZE;
 425     ssize_t count;
 426
 427     if (!virtio_iommu_mr(s, ep_id)) {
 428         return VIRTIO_IOMMU_S_NOENT;
 429     }
 430
 431     count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free);
 432     if (count < 0) {
 433         return VIRTIO_IOMMU_S_INVAL;
 434     }
 435     buf += count;
 436     free -= count;
 437
 438     return VIRTIO_IOMMU_S_OK;
 439 }
 440
 441 static int virtio_iommu_iov_to_req(struct iovec *iov,
 442                                    unsigned int iov_cnt,
 443                                    void *req, size_t req_sz)
 444 {
 445     size_t sz, payload_sz = req_sz - sizeof(struct virtio_iommu_req_tail);
 446
 447     sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz);
 448     if (unlikely(sz != payload_sz)) {
 449         return VIRTIO_IOMMU_S_INVAL;
 450     }
 451     return 0;
 452 }
 453
 454 #define virtio_iommu_handle_req(__req)                                  \
 455 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s,                \
 456                                          struct iovec *iov,             \
 457                                          unsigned int iov_cnt)          \
 458 {                                                                       \
 459     struct virtio_iommu_req_ ## __req req;                              \
 460     int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); \
 461                                                                         \
 462     return ret ? ret : virtio_iommu_ ## __req(s, &req);                 \
 463 }
 464
 465 virtio_iommu_handle_req(attach)
 466 virtio_iommu_handle_req(detach)
 467 virtio_iommu_handle_req(map)
 468 virtio_iommu_handle_req(unmap)
 469
 470 static int virtio_iommu_handle_probe(VirtIOIOMMU *s,
 471                                      struct iovec *iov,
 472                                      unsigned int iov_cnt,
 473                                      uint8_t *buf)
 474 {
 475     struct virtio_iommu_req_probe req;
 476     int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req));
 477
 478     return ret ? ret : virtio_iommu_probe(s, &req, buf);
 479 }
 480
 481 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
 482 {
 483     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
 484     struct virtio_iommu_req_head head;
 485     struct virtio_iommu_req_tail tail = {};
 486     size_t output_size = sizeof(tail), sz;
 487     VirtQueueElement *elem;
 488     unsigned int iov_cnt;
 489     struct iovec *iov;
 490     void *buf = NULL;
 491
 492     for (;;) {
 493         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 494         if (!elem) {
 495             return;
 496         }
 497
 498         if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) ||
 499             iov_size(elem->out_sg, elem->out_num) < sizeof(head)) {
 500             virtio_error(vdev, "virtio-iommu bad head/tail size");
 501             virtqueue_detach_element(vq, elem, 0);
 502             g_free(elem);
 503             break;
 504         }
 505
 506         iov_cnt = elem->out_num;
 507         iov = elem->out_sg;
 508         sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head));
 509         if (unlikely(sz != sizeof(head))) {
 510             tail.status = VIRTIO_IOMMU_S_DEVERR;
 511             goto out;
 512         }
 513         qemu_mutex_lock(&s->mutex);
 514         switch (head.type) {
 515         case VIRTIO_IOMMU_T_ATTACH:
 516             tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt);
 517             break;
 518         case VIRTIO_IOMMU_T_DETACH:
 519             tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt);
 520             break;
 521         case VIRTIO_IOMMU_T_MAP:
 522             tail.status = virtio_iommu_handle_map(s, iov, iov_cnt);
 523             break;
 524         case VIRTIO_IOMMU_T_UNMAP:
 525             tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt);
 526             break;
 527         case VIRTIO_IOMMU_T_PROBE:
 528         {
 529             struct virtio_iommu_req_tail *ptail;
 530
 531             output_size = s->config.probe_size + sizeof(tail);
 532             buf = g_malloc0(output_size);
 533
 534             ptail = (struct virtio_iommu_req_tail *)
 535                         (buf + s->config.probe_size);
 536             ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf);
 537             break;
 538         }
 539         default:
 540             tail.status = VIRTIO_IOMMU_S_UNSUPP;
 541         }
 542         qemu_mutex_unlock(&s->mutex);
 543
 544 out:
 545         sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
 546                           buf ? buf : &tail, output_size);
 547         assert(sz == output_size);
 548
 549         virtqueue_push(vq, elem, sz);
 550         virtio_notify(vdev, vq);
 551         g_free(elem);
 552         g_free(buf);
 553     }
 554 }
 555
 556 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason,
 557                                       int flags, uint32_t endpoint,
 558                                       uint64_t address)
 559 {
 560     VirtIODevice *vdev = &viommu->parent_obj;
 561     VirtQueue *vq = viommu->event_vq;
 562     struct virtio_iommu_fault fault;
 563     VirtQueueElement *elem;
 564     size_t sz;
 565
 566     memset(&fault, 0, sizeof(fault));
 567     fault.reason = reason;
 568     fault.flags = cpu_to_le32(flags);
 569     fault.endpoint = cpu_to_le32(endpoint);
 570     fault.address = cpu_to_le64(address);
 571
 572     elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 573
 574     if (!elem) {
 575         error_report_once(
 576             "no buffer available in event queue to report event");
 577         return;
 578     }
 579
 580     if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) {
 581         virtio_error(vdev, "error buffer of wrong size");
 582         virtqueue_detach_element(vq, elem, 0);
 583         g_free(elem);
 584         return;
 585     }
 586
 587     sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
 588                       &fault, sizeof(fault));
 589     assert(sz == sizeof(fault));
 590
 591     trace_virtio_iommu_report_fault(reason, flags, endpoint, address);
 592     virtqueue_push(vq, elem, sz);
 593     virtio_notify(vdev, vq);
 594     g_free(elem);
 595
 596 }
 597
 598 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
 599                                             IOMMUAccessFlags flag,
 600                                             int iommu_idx)
 601 {
 602     IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
 603     VirtIOIOMMUInterval interval, *mapping_key;
 604     VirtIOIOMMUMapping *mapping_value;
 605     VirtIOIOMMU *s = sdev->viommu;
 606     bool read_fault, write_fault;
 607     VirtIOIOMMUEndpoint *ep;
 608     uint32_t sid, flags;
 609     bool bypass_allowed;
 610     bool found;
 611     int i;
 612
 613     interval.low = addr;
 614     interval.high = addr + 1;
 615
 616     IOMMUTLBEntry entry = {
 617         .target_as = &address_space_memory,
 618         .iova = addr,
 619         .translated_addr = addr,
 620         .addr_mask = (1 << ctz32(s->config.page_size_mask)) - 1,
 621         .perm = IOMMU_NONE,
 622     };
 623
 624     bypass_allowed = virtio_vdev_has_feature(&s->parent_obj,
 625                                              VIRTIO_IOMMU_F_BYPASS);
 626
 627     sid = virtio_iommu_get_bdf(sdev);
 628
 629     trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag);
 630     qemu_mutex_lock(&s->mutex);
 631
 632     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
 633     if (!ep) {
 634         if (!bypass_allowed) {
 635             error_report_once("%s sid=%d is not known!!", __func__, sid);
 636             virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN,
 637                                       VIRTIO_IOMMU_FAULT_F_ADDRESS,
 638                                       sid, addr);
 639         } else {
 640             entry.perm = flag;
 641         }
 642         goto unlock;
 643     }
 644
 645     for (i = 0; i < s->nb_reserved_regions; i++) {
 646         ReservedRegion *reg = &s->reserved_regions[i];
 647
 648         if (addr >= reg->low && addr <= reg->high) {
 649             switch (reg->type) {
 650             case VIRTIO_IOMMU_RESV_MEM_T_MSI:
 651                 entry.perm = flag;
 652                 break;
 653             case VIRTIO_IOMMU_RESV_MEM_T_RESERVED:
 654             default:
 655                 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
 656                                           VIRTIO_IOMMU_FAULT_F_ADDRESS,
 657                                           sid, addr);
 658                 break;
 659             }
 660             goto unlock;
 661         }
 662     }
 663
 664     if (!ep->domain) {
 665         if (!bypass_allowed) {
 666             error_report_once("%s %02x:%02x.%01x not attached to any domain",
 667                               __func__, PCI_BUS_NUM(sid),
 668                               PCI_SLOT(sid), PCI_FUNC(sid));
 669             virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN,
 670                                       VIRTIO_IOMMU_FAULT_F_ADDRESS,
 671                                       sid, addr);
 672         } else {
 673             entry.perm = flag;
 674         }
 675         goto unlock;
 676     }
 677
 678     found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval),
 679                                    (void **)&mapping_key,
 680                                    (void **)&mapping_value);
 681     if (!found) {
 682         error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d",
 683                           __func__, addr, sid);
 684         virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
 685                                   VIRTIO_IOMMU_FAULT_F_ADDRESS,
 686                                   sid, addr);
 687         goto unlock;
 688     }
 689
 690     read_fault = (flag & IOMMU_RO) &&
 691                     !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ);
 692     write_fault = (flag & IOMMU_WO) &&
 693                     !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE);
 694
 695     flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0;
 696     flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0;
 697     if (flags) {
 698         error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d",
 699                           __func__, addr, flag, mapping_value->flags);
 700         flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS;
 701         virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
 702                                   flags | VIRTIO_IOMMU_FAULT_F_ADDRESS,
 703                                   sid, addr);
 704         goto unlock;
 705     }
 706     entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr;
 707     entry.perm = flag;
 708     trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid);
 709
 710 unlock:
 711     qemu_mutex_unlock(&s->mutex);
 712     return entry;
 713 }
 714
 715 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data)
 716 {
 717     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
 718     struct virtio_iommu_config *config = &dev->config;
 719
 720     trace_virtio_iommu_get_config(config->page_size_mask,
 721                                   config->input_range.start,
 722                                   config->input_range.end,
 723                                   config->domain_range.end,
 724                                   config->probe_size);
 725     memcpy(config_data, &dev->config, sizeof(struct virtio_iommu_config));
 726 }
 727
 728 static void virtio_iommu_set_config(VirtIODevice *vdev,
 729                                       const uint8_t *config_data)
 730 {
 731     struct virtio_iommu_config config;
 732
 733     memcpy(&config, config_data, sizeof(struct virtio_iommu_config));
 734     trace_virtio_iommu_set_config(config.page_size_mask,
 735                                   config.input_range.start,
 736                                   config.input_range.end,
 737                                   config.domain_range.end,
 738                                   config.probe_size);
 739 }
 740
 741 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f,
 742                                           Error **errp)
 743 {
 744     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
 745
 746     f |= dev->features;
 747     trace_virtio_iommu_get_features(f);
 748     return f;
 749 }
 750
 751 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
 752 {
 753     guint ua = GPOINTER_TO_UINT(a);
 754     guint ub = GPOINTER_TO_UINT(b);
 755     return (ua > ub) - (ua < ub);
 756 }
 757
 758 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
 759 {
 760     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 761     VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
 762
 763     virtio_init(vdev, "virtio-iommu", VIRTIO_ID_IOMMU,
 764                 sizeof(struct virtio_iommu_config));
 765
 766     memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num));
 767
 768     s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE,
 769                              virtio_iommu_handle_command);
 770     s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL);
 771
 772     s->config.page_size_mask = TARGET_PAGE_MASK;
 773     s->config.input_range.end = -1UL;
 774     s->config.domain_range.end = 32;
 775     s->config.probe_size = VIOMMU_PROBE_SIZE;
 776
 777     virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX);
 778     virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC);
 779     virtio_add_feature(&s->features, VIRTIO_F_VERSION_1);
 780     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE);
 781     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE);
 782     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP);
 783     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS);
 784     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO);
 785     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE);
 786
 787     qemu_mutex_init(&s->mutex);
 788
 789     s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free);
 790
 791     if (s->primary_bus) {
 792         pci_setup_iommu(s->primary_bus, virtio_iommu_find_add_as, s);
 793     } else {
 794         error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!");
 795     }
 796 }
 797
 798 static void virtio_iommu_device_unrealize(DeviceState *dev)
 799 {
 800     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 801     VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
 802
 803     g_hash_table_destroy(s->as_by_busptr);
 804     if (s->domains) {
 805         g_tree_destroy(s->domains);
 806     }
 807     if (s->endpoints) {
 808         g_tree_destroy(s->endpoints);
 809     }
 810
 811     virtio_delete_queue(s->req_vq);
 812     virtio_delete_queue(s->event_vq);
 813     virtio_cleanup(vdev);
 814 }
 815
 816 static void virtio_iommu_device_reset(VirtIODevice *vdev)
 817 {
 818     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
 819
 820     trace_virtio_iommu_device_reset();
 821
 822     if (s->domains) {
 823         g_tree_destroy(s->domains);
 824     }
 825     if (s->endpoints) {
 826         g_tree_destroy(s->endpoints);
 827     }
 828     s->domains = g_tree_new_full((GCompareDataFunc)int_cmp,
 829                                  NULL, NULL, virtio_iommu_put_domain);
 830     s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp,
 831                                    NULL, NULL, virtio_iommu_put_endpoint);
 832 }
 833
 834 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status)
 835 {
 836     trace_virtio_iommu_device_status(status);
 837 }
 838
 839 static void virtio_iommu_instance_init(Object *obj)
 840 {
 841 }
 842
 843 #define VMSTATE_INTERVAL                               \
 844 {                                                      \
 845     .name = "interval",                                \
 846     .version_id = 1,                                   \
 847     .minimum_version_id = 1,                           \
 848     .fields = (VMStateField[]) {                       \
 849         VMSTATE_UINT64(low, VirtIOIOMMUInterval),      \
 850         VMSTATE_UINT64(high, VirtIOIOMMUInterval),     \
 851         VMSTATE_END_OF_LIST()                          \
 852     }                                                  \
 853 }
 854
 855 #define VMSTATE_MAPPING                               \
 856 {                                                     \
 857     .name = "mapping",                                \
 858     .version_id = 1,                                  \
 859     .minimum_version_id = 1,                          \
 860     .fields = (VMStateField[]) {                      \
 861         VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\
 862         VMSTATE_UINT32(flags, VirtIOIOMMUMapping),    \
 863         VMSTATE_END_OF_LIST()                         \
 864     },                                                \
 865 }
 866
 867 static const VMStateDescription vmstate_interval_mapping[2] = {
 868     VMSTATE_MAPPING,   /* value */
 869     VMSTATE_INTERVAL   /* key   */
 870 };
 871
 872 static int domain_preload(void *opaque)
 873 {
 874     VirtIOIOMMUDomain *domain = opaque;
 875
 876     domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
 877                                        NULL, g_free, g_free);
 878     return 0;
 879 }
 880
 881 static const VMStateDescription vmstate_endpoint = {
 882     .name = "endpoint",
 883     .version_id = 1,
 884     .minimum_version_id = 1,
 885     .fields = (VMStateField[]) {
 886         VMSTATE_UINT32(id, VirtIOIOMMUEndpoint),
 887         VMSTATE_END_OF_LIST()
 888     }
 889 };
 890
 891 static const VMStateDescription vmstate_domain = {
 892     .name = "domain",
 893     .version_id = 1,
 894     .minimum_version_id = 1,
 895     .pre_load = domain_preload,
 896     .fields = (VMStateField[]) {
 897         VMSTATE_UINT32(id, VirtIOIOMMUDomain),
 898         VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1,
 899                         vmstate_interval_mapping,
 900                         VirtIOIOMMUInterval, VirtIOIOMMUMapping),
 901         VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1,
 902                         vmstate_endpoint, VirtIOIOMMUEndpoint, next),
 903         VMSTATE_END_OF_LIST()
 904     }
 905 };
 906
 907 static gboolean reconstruct_endpoints(gpointer key, gpointer value,
 908                                       gpointer data)
 909 {
 910     VirtIOIOMMU *s = (VirtIOIOMMU *)data;
 911     VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value;
 912     VirtIOIOMMUEndpoint *iter;
 913
 914     QLIST_FOREACH(iter, &d->endpoint_list, next) {
 915         iter->domain = d;
 916         g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter);
 917     }
 918     return false; /* continue the domain traversal */
 919 }
 920
 921 static int iommu_post_load(void *opaque, int version_id)
 922 {
 923     VirtIOIOMMU *s = opaque;
 924
 925     g_tree_foreach(s->domains, reconstruct_endpoints, s);
 926     return 0;
 927 }
 928
 929 static const VMStateDescription vmstate_virtio_iommu_device = {
 930     .name = "virtio-iommu-device",
 931     .minimum_version_id = 1,
 932     .version_id = 1,
 933     .post_load = iommu_post_load,
 934     .fields = (VMStateField[]) {
 935         VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 1,
 936                                    &vmstate_domain, VirtIOIOMMUDomain),
 937         VMSTATE_END_OF_LIST()
 938     },
 939 };
 940
 941 static const VMStateDescription vmstate_virtio_iommu = {
 942     .name = "virtio-iommu",
 943     .minimum_version_id = 1,
 944     .priority = MIG_PRI_IOMMU,
 945     .version_id = 1,
 946     .fields = (VMStateField[]) {
 947         VMSTATE_VIRTIO_DEVICE,
 948         VMSTATE_END_OF_LIST()
 949     },
 950 };
 951
 952 static Property virtio_iommu_properties[] = {
 953     DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus, "PCI", PCIBus *),
 954     DEFINE_PROP_END_OF_LIST(),
 955 };
 956
 957 static void virtio_iommu_class_init(ObjectClass *klass, void *data)
 958 {
 959     DeviceClass *dc = DEVICE_CLASS(klass);
 960     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 961
 962     device_class_set_props(dc, virtio_iommu_properties);
 963     dc->vmsd = &vmstate_virtio_iommu;
 964
 965     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 966     vdc->realize = virtio_iommu_device_realize;
 967     vdc->unrealize = virtio_iommu_device_unrealize;
 968     vdc->reset = virtio_iommu_device_reset;
 969     vdc->get_config = virtio_iommu_get_config;
 970     vdc->set_config = virtio_iommu_set_config;
 971     vdc->get_features = virtio_iommu_get_features;
 972     vdc->set_status = virtio_iommu_set_status;
 973     vdc->vmsd = &vmstate_virtio_iommu_device;
 974 }
 975
 976 static void virtio_iommu_memory_region_class_init(ObjectClass *klass,
 977                                                   void *data)
 978 {
 979     IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
 980
 981     imrc->translate = virtio_iommu_translate;
 982 }
 983
 984 static const TypeInfo virtio_iommu_info = {
 985     .name = TYPE_VIRTIO_IOMMU,
 986     .parent = TYPE_VIRTIO_DEVICE,
 987     .instance_size = sizeof(VirtIOIOMMU),
 988     .instance_init = virtio_iommu_instance_init,
 989     .class_init = virtio_iommu_class_init,
 990 };
 991
 992 static const TypeInfo virtio_iommu_memory_region_info = {
 993     .parent = TYPE_IOMMU_MEMORY_REGION,
 994     .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION,
 995     .class_init = virtio_iommu_memory_region_class_init,
 996 };
 997
 998 static void virtio_register_types(void)
 999 {
1000     type_register_static(&virtio_iommu_info);
1001     type_register_static(&virtio_iommu_memory_region_info);
1002 }
1003
1004 type_init(virtio_register_types)