drivers/dma/idxd/init.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/slab.h>
   7 #include <linux/pci.h>
   8 #include <linux/interrupt.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/workqueue.h>
  12 #include <linux/aer.h>
  13 #include <linux/fs.h>
  14 #include <linux/io-64-nonatomic-lo-hi.h>
  15 #include <linux/device.h>
  16 #include <linux/idr.h>
  17 #include <linux/intel-svm.h>
  18 #include <linux/iommu.h>
  19 #include <uapi/linux/idxd.h>
  20 #include <linux/dmaengine.h>
  21 #include "../dmaengine.h"
  22 #include "registers.h"
  23 #include "idxd.h"
  24
  25 MODULE_VERSION(IDXD_DRIVER_VERSION);
  26 MODULE_LICENSE("GPL v2");
  27 MODULE_AUTHOR("Intel Corporation");
  28
  29 #define DRV_NAME "idxd"
  30
  31 bool support_enqcmd;
  32
  33 static struct idr idxd_idrs[IDXD_TYPE_MAX];
  34 static struct mutex idxd_idr_lock;
  35
  36 static struct pci_device_id idxd_pci_tbl[] = {
  37         /* DSA ver 1.0 platforms */
  38         { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) },
  39
  40         /* IAX ver 1.0 platforms */
  41         { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) },
  42         { 0, }
  43 };
  44 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
  45
  46 static char *idxd_name[] = {
  47         "dsa",
  48         "iax"
  49 };
  50
  51 const char *idxd_get_dev_name(struct idxd_device *idxd)
  52 {
  53         return idxd_name[idxd->type];
  54 }
  55
  56 static int idxd_setup_interrupts(struct idxd_device *idxd)
  57 {
  58         struct pci_dev *pdev = idxd->pdev;
  59         struct device *dev = &pdev->dev;
  60         struct msix_entry *msix;
  61         struct idxd_irq_entry *irq_entry;
  62         int i, msixcnt;
  63         int rc = 0;
  64         union msix_perm mperm;
  65
  66         msixcnt = pci_msix_vec_count(pdev);
  67         if (msixcnt < 0) {
  68                 dev_err(dev, "Not MSI-X interrupt capable.\n");
  69                 goto err_no_irq;
  70         }
  71
  72         idxd->msix_entries = devm_kzalloc(dev, sizeof(struct msix_entry) *
  73                         msixcnt, GFP_KERNEL);
  74         if (!idxd->msix_entries) {
  75                 rc = -ENOMEM;
  76                 goto err_no_irq;
  77         }
  78
  79         for (i = 0; i < msixcnt; i++)
  80                 idxd->msix_entries[i].entry = i;
  81
  82         rc = pci_enable_msix_exact(pdev, idxd->msix_entries, msixcnt);
  83         if (rc) {
  84                 dev_err(dev, "Failed enabling %d MSIX entries.\n", msixcnt);
  85                 goto err_no_irq;
  86         }
  87         dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
  88
  89         /*
  90          * We implement 1 completion list per MSI-X entry except for
  91          * entry 0, which is for errors and others.
  92          */
  93         idxd->irq_entries = devm_kcalloc(dev, msixcnt,
  94                                          sizeof(struct idxd_irq_entry),
  95                                          GFP_KERNEL);
  96         if (!idxd->irq_entries) {
  97                 rc = -ENOMEM;
  98                 goto err_no_irq;
  99         }
 100
 101         for (i = 0; i < msixcnt; i++) {
 102                 idxd->irq_entries[i].id = i;
 103                 idxd->irq_entries[i].idxd = idxd;
 104                 spin_lock_init(&idxd->irq_entries[i].list_lock);
 105         }
 106
 107         msix = &idxd->msix_entries[0];
 108         irq_entry = &idxd->irq_entries[0];
 109         rc = devm_request_threaded_irq(dev, msix->vector, idxd_irq_handler,
 110                                        idxd_misc_thread, 0, "idxd-misc",
 111                                        irq_entry);
 112         if (rc < 0) {
 113                 dev_err(dev, "Failed to allocate misc interrupt.\n");
 114                 goto err_no_irq;
 115         }
 116
 117         dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n",
 118                 msix->vector);
 119
 120         /* first MSI-X entry is not for wq interrupts */
 121         idxd->num_wq_irqs = msixcnt - 1;
 122
 123         for (i = 1; i < msixcnt; i++) {
 124                 msix = &idxd->msix_entries[i];
 125                 irq_entry = &idxd->irq_entries[i];
 126
 127                 init_llist_head(&idxd->irq_entries[i].pending_llist);
 128                 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list);
 129                 rc = devm_request_threaded_irq(dev, msix->vector,
 130                                                idxd_irq_handler,
 131                                                idxd_wq_thread, 0,
 132                                                "idxd-portal", irq_entry);
 133                 if (rc < 0) {
 134                         dev_err(dev, "Failed to allocate irq %d.\n",
 135                                 msix->vector);
 136                         goto err_no_irq;
 137                 }
 138                 dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n",
 139                         i, msix->vector);
 140         }
 141
 142         idxd_unmask_error_interrupts(idxd);
 143
 144         /* Setup MSIX permission table */
 145         mperm.bits = 0;
 146         mperm.pasid = idxd->pasid;
 147         mperm.pasid_en = device_pasid_enabled(idxd);
 148         for (i = 1; i < msixcnt; i++)
 149                 iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8);
 150
 151         return 0;
 152
 153  err_no_irq:
 154         /* Disable error interrupt generation */
 155         idxd_mask_error_interrupts(idxd);
 156         pci_disable_msix(pdev);
 157         dev_err(dev, "No usable interrupts\n");
 158         return rc;
 159 }
 160
 161 static int idxd_setup_internals(struct idxd_device *idxd)
 162 {
 163         struct device *dev = &idxd->pdev->dev;
 164         int i;
 165
 166         init_waitqueue_head(&idxd->cmd_waitq);
 167         idxd->groups = devm_kcalloc(dev, idxd->max_groups,
 168                                     sizeof(struct idxd_group), GFP_KERNEL);
 169         if (!idxd->groups)
 170                 return -ENOMEM;
 171
 172         for (i = 0; i < idxd->max_groups; i++) {
 173                 idxd->groups[i].idxd = idxd;
 174                 idxd->groups[i].id = i;
 175                 idxd->groups[i].tc_a = -1;
 176                 idxd->groups[i].tc_b = -1;
 177         }
 178
 179         idxd->wqs = devm_kcalloc(dev, idxd->max_wqs, sizeof(struct idxd_wq),
 180                                  GFP_KERNEL);
 181         if (!idxd->wqs)
 182                 return -ENOMEM;
 183
 184         idxd->engines = devm_kcalloc(dev, idxd->max_engines,
 185                                      sizeof(struct idxd_engine), GFP_KERNEL);
 186         if (!idxd->engines)
 187                 return -ENOMEM;
 188
 189         for (i = 0; i < idxd->max_wqs; i++) {
 190                 struct idxd_wq *wq = &idxd->wqs[i];
 191
 192                 wq->id = i;
 193                 wq->idxd = idxd;
 194                 mutex_init(&wq->wq_lock);
 195                 wq->idxd_cdev.minor = -1;
 196                 wq->max_xfer_bytes = idxd->max_xfer_bytes;
 197                 wq->max_batch_size = idxd->max_batch_size;
 198                 wq->wqcfg = devm_kzalloc(dev, idxd->wqcfg_size, GFP_KERNEL);
 199                 if (!wq->wqcfg)
 200                         return -ENOMEM;
 201         }
 202
 203         for (i = 0; i < idxd->max_engines; i++) {
 204                 idxd->engines[i].idxd = idxd;
 205                 idxd->engines[i].id = i;
 206         }
 207
 208         idxd->wq = create_workqueue(dev_name(dev));
 209         if (!idxd->wq)
 210                 return -ENOMEM;
 211
 212         return 0;
 213 }
 214
 215 static void idxd_read_table_offsets(struct idxd_device *idxd)
 216 {
 217         union offsets_reg offsets;
 218         struct device *dev = &idxd->pdev->dev;
 219
 220         offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
 221         offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
 222         idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
 223         dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
 224         idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
 225         dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
 226         idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
 227         dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
 228         idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
 229         dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
 230 }
 231
 232 static void idxd_read_caps(struct idxd_device *idxd)
 233 {
 234         struct device *dev = &idxd->pdev->dev;
 235         int i;
 236
 237         /* reading generic capabilities */
 238         idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
 239         dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
 240         idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
 241         dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
 242         idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift;
 243         dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
 244         if (idxd->hw.gen_cap.config_en)
 245                 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
 246
 247         /* reading group capabilities */
 248         idxd->hw.group_cap.bits =
 249                 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET);
 250         dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
 251         idxd->max_groups = idxd->hw.group_cap.num_groups;
 252         dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
 253         idxd->max_tokens = idxd->hw.group_cap.total_tokens;
 254         dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens);
 255         idxd->nr_tokens = idxd->max_tokens;
 256
 257         /* read engine capabilities */
 258         idxd->hw.engine_cap.bits =
 259                 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET);
 260         dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits);
 261         idxd->max_engines = idxd->hw.engine_cap.num_engines;
 262         dev_dbg(dev, "max engines: %u\n", idxd->max_engines);
 263
 264         /* read workqueue capabilities */
 265         idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET);
 266         dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits);
 267         idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size;
 268         dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size);
 269         idxd->max_wqs = idxd->hw.wq_cap.num_wqs;
 270         dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs);
 271         idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN);
 272         dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size);
 273
 274         /* reading operation capabilities */
 275         for (i = 0; i < 4; i++) {
 276                 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base +
 277                                 IDXD_OPCAP_OFFSET + i * sizeof(u64));
 278                 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
 279         }
 280 }
 281
 282 static struct idxd_device *idxd_alloc(struct pci_dev *pdev)
 283 {
 284         struct device *dev = &pdev->dev;
 285         struct idxd_device *idxd;
 286
 287         idxd = devm_kzalloc(dev, sizeof(struct idxd_device), GFP_KERNEL);
 288         if (!idxd)
 289                 return NULL;
 290
 291         idxd->pdev = pdev;
 292         spin_lock_init(&idxd->dev_lock);
 293
 294         return idxd;
 295 }
 296
 297 static int idxd_enable_system_pasid(struct idxd_device *idxd)
 298 {
 299         int flags;
 300         unsigned int pasid;
 301         struct iommu_sva *sva;
 302
 303         flags = SVM_FLAG_SUPERVISOR_MODE;
 304
 305         sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
 306         if (IS_ERR(sva)) {
 307                 dev_warn(&idxd->pdev->dev,
 308                          "iommu sva bind failed: %ld\n", PTR_ERR(sva));
 309                 return PTR_ERR(sva);
 310         }
 311
 312         pasid = iommu_sva_get_pasid(sva);
 313         if (pasid == IOMMU_PASID_INVALID) {
 314                 iommu_sva_unbind_device(sva);
 315                 return -ENODEV;
 316         }
 317
 318         idxd->sva = sva;
 319         idxd->pasid = pasid;
 320         dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
 321         return 0;
 322 }
 323
 324 static void idxd_disable_system_pasid(struct idxd_device *idxd)
 325 {
 326
 327         iommu_sva_unbind_device(idxd->sva);
 328         idxd->sva = NULL;
 329 }
 330
 331 static int idxd_probe(struct idxd_device *idxd)
 332 {
 333         struct pci_dev *pdev = idxd->pdev;
 334         struct device *dev = &pdev->dev;
 335         int rc;
 336
 337         dev_dbg(dev, "%s entered and resetting device\n", __func__);
 338         rc = idxd_device_init_reset(idxd);
 339         if (rc < 0)
 340                 return rc;
 341
 342         dev_dbg(dev, "IDXD reset complete\n");
 343
 344         if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) {
 345                 rc = idxd_enable_system_pasid(idxd);
 346                 if (rc < 0)
 347                         dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
 348                 else
 349                         set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
 350         }
 351
 352         idxd_read_caps(idxd);
 353         idxd_read_table_offsets(idxd);
 354
 355         rc = idxd_setup_internals(idxd);
 356         if (rc)
 357                 goto err_setup;
 358
 359         rc = idxd_setup_interrupts(idxd);
 360         if (rc)
 361                 goto err_setup;
 362
 363         dev_dbg(dev, "IDXD interrupt setup complete.\n");
 364
 365         mutex_lock(&idxd_idr_lock);
 366         idxd->id = idr_alloc(&idxd_idrs[idxd->type], idxd, 0, 0, GFP_KERNEL);
 367         mutex_unlock(&idxd_idr_lock);
 368         if (idxd->id < 0) {
 369                 rc = -ENOMEM;
 370                 goto err_idr_fail;
 371         }
 372
 373         idxd->major = idxd_cdev_get_major(idxd);
 374
 375         dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
 376         return 0;
 377
 378  err_idr_fail:
 379         idxd_mask_error_interrupts(idxd);
 380         idxd_mask_msix_vectors(idxd);
 381  err_setup:
 382         if (device_pasid_enabled(idxd))
 383                 idxd_disable_system_pasid(idxd);
 384         return rc;
 385 }
 386
 387 static void idxd_type_init(struct idxd_device *idxd)
 388 {
 389         if (idxd->type == IDXD_TYPE_DSA)
 390                 idxd->compl_size = sizeof(struct dsa_completion_record);
 391         else if (idxd->type == IDXD_TYPE_IAX)
 392                 idxd->compl_size = sizeof(struct iax_completion_record);
 393 }
 394
 395 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 396 {
 397         struct device *dev = &pdev->dev;
 398         struct idxd_device *idxd;
 399         int rc;
 400
 401         rc = pcim_enable_device(pdev);
 402         if (rc)
 403                 return rc;
 404
 405         dev_dbg(dev, "Alloc IDXD context\n");
 406         idxd = idxd_alloc(pdev);
 407         if (!idxd)
 408                 return -ENOMEM;
 409
 410         dev_dbg(dev, "Mapping BARs\n");
 411         idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0);
 412         if (!idxd->reg_base)
 413                 return -ENOMEM;
 414
 415         dev_dbg(dev, "Set DMA masks\n");
 416         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 417         if (rc)
 418                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 419         if (rc)
 420                 return rc;
 421
 422         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
 423         if (rc)
 424                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
 425         if (rc)
 426                 return rc;
 427
 428         idxd_set_type(idxd);
 429
 430         idxd_type_init(idxd);
 431
 432         dev_dbg(dev, "Set PCI master\n");
 433         pci_set_master(pdev);
 434         pci_set_drvdata(pdev, idxd);
 435
 436         idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET);
 437         rc = idxd_probe(idxd);
 438         if (rc) {
 439                 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
 440                 return -ENODEV;
 441         }
 442
 443         rc = idxd_setup_sysfs(idxd);
 444         if (rc) {
 445                 dev_err(dev, "IDXD sysfs setup failed\n");
 446                 return -ENODEV;
 447         }
 448
 449         idxd->state = IDXD_DEV_CONF_READY;
 450
 451         dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
 452                  idxd->hw.version);
 453
 454         return 0;
 455 }
 456
 457 static void idxd_flush_pending_llist(struct idxd_irq_entry *ie)
 458 {
 459         struct idxd_desc *desc, *itr;
 460         struct llist_node *head;
 461
 462         head = llist_del_all(&ie->pending_llist);
 463         if (!head)
 464                 return;
 465
 466         llist_for_each_entry_safe(desc, itr, head, llnode) {
 467                 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
 468                 idxd_free_desc(desc->wq, desc);
 469         }
 470 }
 471
 472 static void idxd_flush_work_list(struct idxd_irq_entry *ie)
 473 {
 474         struct idxd_desc *desc, *iter;
 475
 476         list_for_each_entry_safe(desc, iter, &ie->work_list, list) {
 477                 list_del(&desc->list);
 478                 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
 479                 idxd_free_desc(desc->wq, desc);
 480         }
 481 }
 482
 483 static void idxd_shutdown(struct pci_dev *pdev)
 484 {
 485         struct idxd_device *idxd = pci_get_drvdata(pdev);
 486         int rc, i;
 487         struct idxd_irq_entry *irq_entry;
 488         int msixcnt = pci_msix_vec_count(pdev);
 489
 490         rc = idxd_device_disable(idxd);
 491         if (rc)
 492                 dev_err(&pdev->dev, "Disabling device failed\n");
 493
 494         dev_dbg(&pdev->dev, "%s called\n", __func__);
 495         idxd_mask_msix_vectors(idxd);
 496         idxd_mask_error_interrupts(idxd);
 497
 498         for (i = 0; i < msixcnt; i++) {
 499                 irq_entry = &idxd->irq_entries[i];
 500                 synchronize_irq(idxd->msix_entries[i].vector);
 501                 if (i == 0)
 502                         continue;
 503                 idxd_flush_pending_llist(irq_entry);
 504                 idxd_flush_work_list(irq_entry);
 505         }
 506
 507         destroy_workqueue(idxd->wq);
 508 }
 509
 510 static void idxd_remove(struct pci_dev *pdev)
 511 {
 512         struct idxd_device *idxd = pci_get_drvdata(pdev);
 513
 514         dev_dbg(&pdev->dev, "%s called\n", __func__);
 515         idxd_cleanup_sysfs(idxd);
 516         idxd_shutdown(pdev);
 517         if (device_pasid_enabled(idxd))
 518                 idxd_disable_system_pasid(idxd);
 519         mutex_lock(&idxd_idr_lock);
 520         idr_remove(&idxd_idrs[idxd->type], idxd->id);
 521         mutex_unlock(&idxd_idr_lock);
 522 }
 523
 524 static struct pci_driver idxd_pci_driver = {
 525         .name           = DRV_NAME,
 526         .id_table       = idxd_pci_tbl,
 527         .probe          = idxd_pci_probe,
 528         .remove         = idxd_remove,
 529         .shutdown       = idxd_shutdown,
 530 };
 531
 532 static int __init idxd_init_module(void)
 533 {
 534         int err, i;
 535
 536         /*
 537          * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
 538          * enumerating the device. We can not utilize it.
 539          */
 540         if (!boot_cpu_has(X86_FEATURE_MOVDIR64B)) {
 541                 pr_warn("idxd driver failed to load without MOVDIR64B.\n");
 542                 return -ENODEV;
 543         }
 544
 545         if (!boot_cpu_has(X86_FEATURE_ENQCMD))
 546                 pr_warn("Platform does not have ENQCMD(S) support.\n");
 547         else
 548                 support_enqcmd = true;
 549
 550         mutex_init(&idxd_idr_lock);
 551         for (i = 0; i < IDXD_TYPE_MAX; i++)
 552                 idr_init(&idxd_idrs[i]);
 553
 554         err = idxd_register_bus_type();
 555         if (err < 0)
 556                 return err;
 557
 558         err = idxd_register_driver();
 559         if (err < 0)
 560                 goto err_idxd_driver_register;
 561
 562         err = idxd_cdev_register();
 563         if (err)
 564                 goto err_cdev_register;
 565
 566         err = pci_register_driver(&idxd_pci_driver);
 567         if (err)
 568                 goto err_pci_register;
 569
 570         return 0;
 571
 572 err_pci_register:
 573         idxd_cdev_remove();
 574 err_cdev_register:
 575         idxd_unregister_driver();
 576 err_idxd_driver_register:
 577         idxd_unregister_bus_type();
 578         return err;
 579 }
 580 module_init(idxd_init_module);
 581
 582 static void __exit idxd_exit_module(void)
 583 {
 584         pci_unregister_driver(&idxd_pci_driver);
 585         idxd_cdev_remove();
 586         idxd_unregister_bus_type();
 587 }
 588 module_exit(idxd_exit_module);