drivers/misc/ocxl/pci.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright 2017 IBM Corp.
   3 #include <linux/module.h>
   4 #include <linux/pci.h>
   5 #include <linux/idr.h>
   6 #include <asm/pnv-ocxl.h>
   7 #include "ocxl_internal.h"
   8
   9 /*
  10  * Any opencapi device which wants to use this 'generic' driver should
  11  * use the 0x062B device ID. Vendors should define the subsystem
  12  * vendor/device ID to help differentiate devices.
  13  */
  14 static const struct pci_device_id ocxl_pci_tbl[] = {
  15         { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x062B), },
  16         { }
  17 };
  18 MODULE_DEVICE_TABLE(pci, ocxl_pci_tbl);
  19
  20
  21 static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn)
  22 {
  23         return (get_device(&fn->dev) == NULL) ? NULL : fn;
  24 }
  25
  26 static void ocxl_fn_put(struct ocxl_fn *fn)
  27 {
  28         put_device(&fn->dev);
  29 }
  30
  31 struct ocxl_afu *ocxl_afu_get(struct ocxl_afu *afu)
  32 {
  33         return (get_device(&afu->dev) == NULL) ? NULL : afu;
  34 }
  35
  36 void ocxl_afu_put(struct ocxl_afu *afu)
  37 {
  38         put_device(&afu->dev);
  39 }
  40
  41 static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn)
  42 {
  43         struct ocxl_afu *afu;
  44
  45         afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL);
  46         if (!afu)
  47                 return NULL;
  48
  49         mutex_init(&afu->contexts_lock);
  50         mutex_init(&afu->afu_control_lock);
  51         idr_init(&afu->contexts_idr);
  52         afu->fn = fn;
  53         ocxl_fn_get(fn);
  54         return afu;
  55 }
  56
  57 static void free_afu(struct ocxl_afu *afu)
  58 {
  59         idr_destroy(&afu->contexts_idr);
  60         ocxl_fn_put(afu->fn);
  61         kfree(afu);
  62 }
  63
  64 static void free_afu_dev(struct device *dev)
  65 {
  66         struct ocxl_afu *afu = to_ocxl_afu(dev);
  67
  68         ocxl_unregister_afu(afu);
  69         free_afu(afu);
  70 }
  71
  72 static int set_afu_device(struct ocxl_afu *afu, const char *location)
  73 {
  74         struct ocxl_fn *fn = afu->fn;
  75         int rc;
  76
  77         afu->dev.parent = &fn->dev;
  78         afu->dev.release = free_afu_dev;
  79         rc = dev_set_name(&afu->dev, "%s.%s.%hhu", afu->config.name, location,
  80                 afu->config.idx);
  81         return rc;
  82 }
  83
  84 static int assign_afu_actag(struct ocxl_afu *afu, struct pci_dev *dev)
  85 {
  86         struct ocxl_fn *fn = afu->fn;
  87         int actag_count, actag_offset;
  88
  89         /*
  90          * if there were not enough actags for the function, each afu
  91          * reduces its count as well
  92          */
  93         actag_count = afu->config.actag_supported *
  94                 fn->actag_enabled / fn->actag_supported;
  95         actag_offset = ocxl_actag_afu_alloc(fn, actag_count);
  96         if (actag_offset < 0) {
  97                 dev_err(&afu->dev, "Can't allocate %d actags for AFU: %d\n",
  98                         actag_count, actag_offset);
  99                 return actag_offset;
 100         }
 101         afu->actag_base = fn->actag_base + actag_offset;
 102         afu->actag_enabled = actag_count;
 103
 104         ocxl_config_set_afu_actag(dev, afu->config.dvsec_afu_control_pos,
 105                                 afu->actag_base, afu->actag_enabled);
 106         dev_dbg(&afu->dev, "actag base=%d enabled=%d\n",
 107                 afu->actag_base, afu->actag_enabled);
 108         return 0;
 109 }
 110
 111 static void reclaim_afu_actag(struct ocxl_afu *afu)
 112 {
 113         struct ocxl_fn *fn = afu->fn;
 114         int start_offset, size;
 115
 116         start_offset = afu->actag_base - fn->actag_base;
 117         size = afu->actag_enabled;
 118         ocxl_actag_afu_free(afu->fn, start_offset, size);
 119 }
 120
 121 static int assign_afu_pasid(struct ocxl_afu *afu, struct pci_dev *dev)
 122 {
 123         struct ocxl_fn *fn = afu->fn;
 124         int pasid_count, pasid_offset;
 125
 126         /*
 127          * We only support the case where the function configuration
 128          * requested enough PASIDs to cover all AFUs.
 129          */
 130         pasid_count = 1 << afu->config.pasid_supported_log;
 131         pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count);
 132         if (pasid_offset < 0) {
 133                 dev_err(&afu->dev, "Can't allocate %d PASIDs for AFU: %d\n",
 134                         pasid_count, pasid_offset);
 135                 return pasid_offset;
 136         }
 137         afu->pasid_base = fn->pasid_base + pasid_offset;
 138         afu->pasid_count = 0;
 139         afu->pasid_max = pasid_count;
 140
 141         ocxl_config_set_afu_pasid(dev, afu->config.dvsec_afu_control_pos,
 142                                 afu->pasid_base,
 143                                 afu->config.pasid_supported_log);
 144         dev_dbg(&afu->dev, "PASID base=%d, enabled=%d\n",
 145                 afu->pasid_base, pasid_count);
 146         return 0;
 147 }
 148
 149 static void reclaim_afu_pasid(struct ocxl_afu *afu)
 150 {
 151         struct ocxl_fn *fn = afu->fn;
 152         int start_offset, size;
 153
 154         start_offset = afu->pasid_base - fn->pasid_base;
 155         size = 1 << afu->config.pasid_supported_log;
 156         ocxl_pasid_afu_free(afu->fn, start_offset, size);
 157 }
 158
 159 static int reserve_fn_bar(struct ocxl_fn *fn, int bar)
 160 {
 161         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 162         int rc, idx;
 163
 164         if (bar != 0 && bar != 2 && bar != 4)
 165                 return -EINVAL;
 166
 167         idx = bar >> 1;
 168         if (fn->bar_used[idx]++ == 0) {
 169                 rc = pci_request_region(dev, bar, "ocxl");
 170                 if (rc)
 171                         return rc;
 172         }
 173         return 0;
 174 }
 175
 176 static void release_fn_bar(struct ocxl_fn *fn, int bar)
 177 {
 178         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 179         int idx;
 180
 181         if (bar != 0 && bar != 2 && bar != 4)
 182                 return;
 183
 184         idx = bar >> 1;
 185         if (--fn->bar_used[idx] == 0)
 186                 pci_release_region(dev, bar);
 187         WARN_ON(fn->bar_used[idx] < 0);
 188 }
 189
 190 static int map_mmio_areas(struct ocxl_afu *afu, struct pci_dev *dev)
 191 {
 192         int rc;
 193
 194         rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar);
 195         if (rc)
 196                 return rc;
 197
 198         rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar);
 199         if (rc) {
 200                 release_fn_bar(afu->fn, afu->config.global_mmio_bar);
 201                 return rc;
 202         }
 203
 204         afu->global_mmio_start =
 205                 pci_resource_start(dev, afu->config.global_mmio_bar) +
 206                 afu->config.global_mmio_offset;
 207         afu->pp_mmio_start =
 208                 pci_resource_start(dev, afu->config.pp_mmio_bar) +
 209                 afu->config.pp_mmio_offset;
 210
 211         afu->global_mmio_ptr = ioremap(afu->global_mmio_start,
 212                                 afu->config.global_mmio_size);
 213         if (!afu->global_mmio_ptr) {
 214                 release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
 215                 release_fn_bar(afu->fn, afu->config.global_mmio_bar);
 216                 dev_err(&dev->dev, "Error mapping global mmio area\n");
 217                 return -ENOMEM;
 218         }
 219
 220         /*
 221          * Leave an empty page between the per-process mmio area and
 222          * the AFU interrupt mappings
 223          */
 224         afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE;
 225         return 0;
 226 }
 227
 228 static void unmap_mmio_areas(struct ocxl_afu *afu)
 229 {
 230         if (afu->global_mmio_ptr) {
 231                 iounmap(afu->global_mmio_ptr);
 232                 afu->global_mmio_ptr = NULL;
 233         }
 234         afu->global_mmio_start = 0;
 235         afu->pp_mmio_start = 0;
 236         release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
 237         release_fn_bar(afu->fn, afu->config.global_mmio_bar);
 238 }
 239
 240 static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
 241 {
 242         int rc;
 243
 244         rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx);
 245         if (rc)
 246                 return rc;
 247
 248         rc = set_afu_device(afu, dev_name(&dev->dev));
 249         if (rc)
 250                 return rc;
 251
 252         rc = assign_afu_actag(afu, dev);
 253         if (rc)
 254                 return rc;
 255
 256         rc = assign_afu_pasid(afu, dev);
 257         if (rc) {
 258                 reclaim_afu_actag(afu);
 259                 return rc;
 260         }
 261
 262         rc = map_mmio_areas(afu, dev);
 263         if (rc) {
 264                 reclaim_afu_pasid(afu);
 265                 reclaim_afu_actag(afu);
 266                 return rc;
 267         }
 268         return 0;
 269 }
 270
 271 static void deconfigure_afu(struct ocxl_afu *afu)
 272 {
 273         unmap_mmio_areas(afu);
 274         reclaim_afu_pasid(afu);
 275         reclaim_afu_actag(afu);
 276 }
 277
 278 static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu)
 279 {
 280         int rc;
 281
 282         ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1);
 283         /*
 284          * Char device creation is the last step, as processes can
 285          * call our driver immediately, so all our inits must be finished.
 286          */
 287         rc = ocxl_create_cdev(afu);
 288         if (rc)
 289                 return rc;
 290         return 0;
 291 }
 292
 293 static void deactivate_afu(struct ocxl_afu *afu)
 294 {
 295         struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
 296
 297         ocxl_destroy_cdev(afu);
 298         ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0);
 299 }
 300
 301 static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx)
 302 {
 303         int rc;
 304         struct ocxl_afu *afu;
 305
 306         afu = alloc_afu(fn);
 307         if (!afu)
 308                 return -ENOMEM;
 309
 310         rc = configure_afu(afu, afu_idx, dev);
 311         if (rc) {
 312                 free_afu(afu);
 313                 return rc;
 314         }
 315
 316         rc = ocxl_register_afu(afu);
 317         if (rc)
 318                 goto err;
 319
 320         rc = ocxl_sysfs_add_afu(afu);
 321         if (rc)
 322                 goto err;
 323
 324         rc = activate_afu(dev, afu);
 325         if (rc)
 326                 goto err_sys;
 327
 328         list_add_tail(&afu->list, &fn->afu_list);
 329         return 0;
 330
 331 err_sys:
 332         ocxl_sysfs_remove_afu(afu);
 333 err:
 334         deconfigure_afu(afu);
 335         device_unregister(&afu->dev);
 336         return rc;
 337 }
 338
 339 static void remove_afu(struct ocxl_afu *afu)
 340 {
 341         list_del(&afu->list);
 342         ocxl_context_detach_all(afu);
 343         deactivate_afu(afu);
 344         ocxl_sysfs_remove_afu(afu);
 345         deconfigure_afu(afu);
 346         device_unregister(&afu->dev);
 347 }
 348
 349 static struct ocxl_fn *alloc_function(struct pci_dev *dev)
 350 {
 351         struct ocxl_fn *fn;
 352
 353         fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL);
 354         if (!fn)
 355                 return NULL;
 356
 357         INIT_LIST_HEAD(&fn->afu_list);
 358         INIT_LIST_HEAD(&fn->pasid_list);
 359         INIT_LIST_HEAD(&fn->actag_list);
 360         return fn;
 361 }
 362
 363 static void free_function(struct ocxl_fn *fn)
 364 {
 365         WARN_ON(!list_empty(&fn->afu_list));
 366         WARN_ON(!list_empty(&fn->pasid_list));
 367         kfree(fn);
 368 }
 369
 370 static void free_function_dev(struct device *dev)
 371 {
 372         struct ocxl_fn *fn = to_ocxl_function(dev);
 373
 374         free_function(fn);
 375 }
 376
 377 static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev)
 378 {
 379         int rc;
 380
 381         fn->dev.parent = &dev->dev;
 382         fn->dev.release = free_function_dev;
 383         rc = dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev));
 384         if (rc)
 385                 return rc;
 386         pci_set_drvdata(dev, fn);
 387         return 0;
 388 }
 389
 390 static int assign_function_actag(struct ocxl_fn *fn)
 391 {
 392         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 393         u16 base, enabled, supported;
 394         int rc;
 395
 396         rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported);
 397         if (rc)
 398                 return rc;
 399
 400         fn->actag_base = base;
 401         fn->actag_enabled = enabled;
 402         fn->actag_supported = supported;
 403
 404         ocxl_config_set_actag(dev, fn->config.dvsec_function_pos,
 405                         fn->actag_base, fn->actag_enabled);
 406         dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n",
 407                 fn->actag_base, fn->actag_enabled);
 408         return 0;
 409 }
 410
 411 static int set_function_pasid(struct ocxl_fn *fn)
 412 {
 413         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 414         int rc, desired_count, max_count;
 415
 416         /* A function may not require any PASID */
 417         if (fn->config.max_pasid_log < 0)
 418                 return 0;
 419
 420         rc = ocxl_config_get_pasid_info(dev, &max_count);
 421         if (rc)
 422                 return rc;
 423
 424         desired_count = 1 << fn->config.max_pasid_log;
 425
 426         if (desired_count > max_count) {
 427                 dev_err(&fn->dev,
 428                         "Function requires more PASIDs than is available (%d vs. %d)\n",
 429                         desired_count, max_count);
 430                 return -ENOSPC;
 431         }
 432
 433         fn->pasid_base = 0;
 434         return 0;
 435 }
 436
 437 static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev)
 438 {
 439         int rc;
 440
 441         rc = pci_enable_device(dev);
 442         if (rc) {
 443                 dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc);
 444                 return rc;
 445         }
 446
 447         /*
 448          * Once it has been confirmed to work on our hardware, we
 449          * should reset the function, to force the adapter to restart
 450          * from scratch.
 451          * A function reset would also reset all its AFUs.
 452          *
 453          * Some hints for implementation:
 454          *
 455          * - there's not status bit to know when the reset is done. We
 456          *   should try reading the config space to know when it's
 457          *   done.
 458          * - probably something like:
 459          *      Reset
 460          *      wait 100ms
 461          *      issue config read
 462          *      allow device up to 1 sec to return success on config
 463          *      read before declaring it broken
 464          *
 465          * Some shared logic on the card (CFG, TLX) won't be reset, so
 466          * there's no guarantee that it will be enough.
 467          */
 468         rc = ocxl_config_read_function(dev, &fn->config);
 469         if (rc)
 470                 return rc;
 471
 472         rc = set_function_device(fn, dev);
 473         if (rc)
 474                 return rc;
 475
 476         rc = assign_function_actag(fn);
 477         if (rc)
 478                 return rc;
 479
 480         rc = set_function_pasid(fn);
 481         if (rc)
 482                 return rc;
 483
 484         rc = ocxl_link_setup(dev, 0, &fn->link);
 485         if (rc)
 486                 return rc;
 487
 488         rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos);
 489         if (rc) {
 490                 ocxl_link_release(dev, fn->link);
 491                 return rc;
 492         }
 493         return 0;
 494 }
 495
 496 static void deconfigure_function(struct ocxl_fn *fn)
 497 {
 498         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 499
 500         ocxl_link_release(dev, fn->link);
 501         pci_disable_device(dev);
 502 }
 503
 504 static struct ocxl_fn *init_function(struct pci_dev *dev)
 505 {
 506         struct ocxl_fn *fn;
 507         int rc;
 508
 509         fn = alloc_function(dev);
 510         if (!fn)
 511                 return ERR_PTR(-ENOMEM);
 512
 513         rc = configure_function(fn, dev);
 514         if (rc) {
 515                 free_function(fn);
 516                 return ERR_PTR(rc);
 517         }
 518
 519         rc = device_register(&fn->dev);
 520         if (rc) {
 521                 deconfigure_function(fn);
 522                 device_unregister(&fn->dev);
 523                 return ERR_PTR(rc);
 524         }
 525         return fn;
 526 }
 527
 528 static void remove_function(struct ocxl_fn *fn)
 529 {
 530         deconfigure_function(fn);
 531         device_unregister(&fn->dev);
 532 }
 533
 534 static int ocxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
 535 {
 536         int rc, afu_count = 0;
 537         u8 afu;
 538         struct ocxl_fn *fn;
 539
 540         if (!radix_enabled()) {
 541                 dev_err(&dev->dev, "Unsupported memory model (hash)\n");
 542                 return -ENODEV;
 543         }
 544
 545         fn = init_function(dev);
 546         if (IS_ERR(fn)) {
 547                 dev_err(&dev->dev, "function init failed: %li\n",
 548                         PTR_ERR(fn));
 549                 return PTR_ERR(fn);
 550         }
 551
 552         for (afu = 0; afu <= fn->config.max_afu_index; afu++) {
 553                 rc = ocxl_config_check_afu_index(dev, &fn->config, afu);
 554                 if (rc > 0) {
 555                         rc = init_afu(dev, fn, afu);
 556                         if (rc) {
 557                                 dev_err(&dev->dev,
 558                                         "Can't initialize AFU index %d\n", afu);
 559                                 continue;
 560                         }
 561                         afu_count++;
 562                 }
 563         }
 564         dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count);
 565         return 0;
 566 }
 567
 568 static void ocxl_remove(struct pci_dev *dev)
 569 {
 570         struct ocxl_afu *afu, *tmp;
 571         struct ocxl_fn *fn = pci_get_drvdata(dev);
 572
 573         list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) {
 574                 remove_afu(afu);
 575         }
 576         remove_function(fn);
 577 }
 578
 579 struct pci_driver ocxl_pci_driver = {
 580         .name = "ocxl",
 581         .id_table = ocxl_pci_tbl,
 582         .probe = ocxl_probe,
 583         .remove = ocxl_remove,
 584         .shutdown = ocxl_remove,
 585 };