drivers/crypto/cavium/nitrox/nitrox_main.c

   1 #include <linux/aer.h>
   2 #include <linux/delay.h>
   3 #include <linux/debugfs.h>
   4 #include <linux/firmware.h>
   5 #include <linux/list.h>
   6 #include <linux/module.h>
   7 #include <linux/mutex.h>
   8 #include <linux/pci.h>
   9 #include <linux/pci_ids.h>
  10
  11 #include "nitrox_dev.h"
  12 #include "nitrox_common.h"
  13 #include "nitrox_csr.h"
  14
  15 #define CNN55XX_DEV_ID  0x12
  16 #define MAX_PF_QUEUES   64
  17 #define UCODE_HLEN 48
  18 #define SE_GROUP 0
  19
  20 #define DRIVER_VERSION "1.0"
  21 /* SE microcode */
  22 #define SE_FW   "cnn55xx_se.fw"
  23
  24 static const char nitrox_driver_name[] = "CNN55XX";
  25
  26 static LIST_HEAD(ndevlist);
  27 static DEFINE_MUTEX(devlist_lock);
  28 static unsigned int num_devices;
  29
  30 /**
  31  * nitrox_pci_tbl - PCI Device ID Table
  32  */
  33 static const struct pci_device_id nitrox_pci_tbl[] = {
  34         {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
  35         /* required last entry */
  36         {0, }
  37 };
  38 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
  39
  40 static unsigned int qlen = DEFAULT_CMD_QLEN;
  41 module_param(qlen, uint, 0644);
  42 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
  43
  44 /**
  45  * struct ucode - Firmware Header
  46  * @id: microcode ID
  47  * @version: firmware version
  48  * @code_size: code section size
  49  * @raz: alignment
  50  * @code: code section
  51  */
  52 struct ucode {
  53         u8 id;
  54         char version[VERSION_LEN - 1];
  55         __be32 code_size;
  56         u8 raz[12];
  57         u64 code[0];
  58 };
  59
  60 /**
  61  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
  62  */
  63 static void write_to_ucd_unit(struct nitrox_device *ndev,
  64                               struct ucode *ucode)
  65 {
  66         u32 code_size = be32_to_cpu(ucode->code_size) * 2;
  67         u64 offset, data;
  68         int i = 0;
  69
  70         /*
  71          * UCD structure
  72          *
  73          *  -------------
  74          *  |    BLK 7  |
  75          *  -------------
  76          *  |    BLK 6  |
  77          *  -------------
  78          *  |    ...    |
  79          *  -------------
  80          *  |    BLK 0  |
  81          *  -------------
  82          *  Total of 8 blocks, each size 32KB
  83          */
  84
  85         /* set the block number */
  86         offset = UCD_UCODE_LOAD_BLOCK_NUM;
  87         nitrox_write_csr(ndev, offset, 0);
  88
  89         code_size = roundup(code_size, 8);
  90         while (code_size) {
  91                 data = ucode->code[i];
  92                 /* write 8 bytes at a time */
  93                 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
  94                 nitrox_write_csr(ndev, offset, data);
  95                 code_size -= 8;
  96                 i++;
  97         }
  98
  99         /* put all SE cores in group 0 */
 100         offset = POM_GRP_EXECMASKX(SE_GROUP);
 101         nitrox_write_csr(ndev, offset, (~0ULL));
 102
 103         for (i = 0; i < ndev->hw.se_cores; i++) {
 104                 /*
 105                  * write block number and firware length
 106                  * bit:<2:0> block number
 107                  * bit:3 is set SE uses 32KB microcode
 108                  * bit:3 is clear SE uses 64KB microcode
 109                  */
 110                 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
 111                 nitrox_write_csr(ndev, offset, 0x8);
 112         }
 113         usleep_range(300, 400);
 114 }
 115
 116 static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
 117 {
 118         const struct firmware *fw;
 119         struct ucode *ucode;
 120         int ret;
 121
 122         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 123
 124         ret = request_firmware(&fw, fw_name, DEV(ndev));
 125         if (ret < 0) {
 126                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 127                 return ret;
 128         }
 129
 130         ucode = (struct ucode *)fw->data;
 131         /* copy the firmware version */
 132         memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
 133         ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
 134
 135         write_to_ucd_unit(ndev, ucode);
 136         release_firmware(fw);
 137
 138         set_bit(NITROX_UCODE_LOADED, &ndev->status);
 139         /* barrier to sync with other cpus */
 140         smp_mb__after_atomic();
 141         return 0;
 142 }
 143
 144 /**
 145  * nitrox_add_to_devlist - add NITROX device to global device list
 146  * @ndev: NITROX device
 147  */
 148 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
 149 {
 150         struct nitrox_device *dev;
 151         int ret = 0;
 152
 153         INIT_LIST_HEAD(&ndev->list);
 154         refcount_set(&ndev->refcnt, 1);
 155
 156         mutex_lock(&devlist_lock);
 157         list_for_each_entry(dev, &ndevlist, list) {
 158                 if (dev == ndev) {
 159                         ret = -EEXIST;
 160                         goto unlock;
 161                 }
 162         }
 163         ndev->idx = num_devices++;
 164         list_add_tail(&ndev->list, &ndevlist);
 165 unlock:
 166         mutex_unlock(&devlist_lock);
 167         return ret;
 168 }
 169
 170 /**
 171  * nitrox_remove_from_devlist - remove NITROX device from
 172  *   global device list
 173  * @ndev: NITROX device
 174  */
 175 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
 176 {
 177         mutex_lock(&devlist_lock);
 178         list_del(&ndev->list);
 179         num_devices--;
 180         mutex_unlock(&devlist_lock);
 181 }
 182
 183 struct nitrox_device *nitrox_get_first_device(void)
 184 {
 185         struct nitrox_device *ndev = NULL;
 186
 187         mutex_lock(&devlist_lock);
 188         list_for_each_entry(ndev, &ndevlist, list) {
 189                 if (nitrox_ready(ndev))
 190                         break;
 191         }
 192         mutex_unlock(&devlist_lock);
 193         if (!ndev)
 194                 return NULL;
 195
 196         refcount_inc(&ndev->refcnt);
 197         /* barrier to sync with other cpus */
 198         smp_mb__after_atomic();
 199         return ndev;
 200 }
 201
 202 void nitrox_put_device(struct nitrox_device *ndev)
 203 {
 204         if (!ndev)
 205                 return;
 206
 207         refcount_dec(&ndev->refcnt);
 208         /* barrier to sync with other cpus */
 209         smp_mb__after_atomic();
 210 }
 211
 212 static int nitrox_reset_device(struct pci_dev *pdev)
 213 {
 214         int pos = 0;
 215
 216         pos = pci_save_state(pdev);
 217         if (pos) {
 218                 dev_err(&pdev->dev, "Failed to save pci state\n");
 219                 return -ENOMEM;
 220         }
 221
 222         pos = pci_pcie_cap(pdev);
 223         if (!pos)
 224                 return -ENOTTY;
 225
 226         if (!pci_wait_for_pending_transaction(pdev))
 227                 dev_err(&pdev->dev, "waiting for pending transaction\n");
 228
 229         pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
 230         msleep(100);
 231         pci_restore_state(pdev);
 232
 233         return 0;
 234 }
 235
 236 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
 237 {
 238         int err;
 239
 240         err = nitrox_common_sw_init(ndev);
 241         if (err)
 242                 return err;
 243
 244         err = nitrox_pf_init_isr(ndev);
 245         if (err)
 246                 nitrox_common_sw_cleanup(ndev);
 247
 248         return err;
 249 }
 250
 251 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
 252 {
 253         nitrox_pf_cleanup_isr(ndev);
 254         nitrox_common_sw_cleanup(ndev);
 255 }
 256
 257 /**
 258  * nitrox_bist_check - Check NITORX BIST registers status
 259  * @ndev: NITROX device
 260  */
 261 static int nitrox_bist_check(struct nitrox_device *ndev)
 262 {
 263         u64 value = 0;
 264         int i;
 265
 266         for (i = 0; i < NR_CLUSTERS; i++) {
 267                 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
 268                 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
 269         }
 270         value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
 271         value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
 272         value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
 273         value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
 274         value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
 275         value += nitrox_read_csr(ndev, POM_BIST_REG);
 276         value += nitrox_read_csr(ndev, BMI_BIST_REG);
 277         value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
 278         value += nitrox_read_csr(ndev, BMO_BIST_REG);
 279         value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
 280         value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
 281         if (value)
 282                 return -EIO;
 283         return 0;
 284 }
 285
 286 static void nitrox_get_hwinfo(struct nitrox_device *ndev)
 287 {
 288         union emu_fuse_map emu_fuse;
 289         u64 offset;
 290         int i;
 291
 292         for (i = 0; i < NR_CLUSTERS; i++) {
 293                 u8 dead_cores;
 294
 295                 offset = EMU_FUSE_MAPX(i);
 296                 emu_fuse.value = nitrox_read_csr(ndev, offset);
 297                 if (emu_fuse.s.valid) {
 298                         dead_cores = hweight32(emu_fuse.s.ae_fuse);
 299                         ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
 300                         dead_cores = hweight16(emu_fuse.s.se_fuse);
 301                         ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
 302                 }
 303         }
 304 }
 305
 306 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
 307 {
 308         int err;
 309
 310         err = nitrox_bist_check(ndev);
 311         if (err) {
 312                 dev_err(&ndev->pdev->dev, "BIST check failed\n");
 313                 return err;
 314         }
 315         /* get cores information */
 316         nitrox_get_hwinfo(ndev);
 317
 318         nitrox_config_nps_unit(ndev);
 319         nitrox_config_pom_unit(ndev);
 320         nitrox_config_efl_unit(ndev);
 321         /* configure IO units */
 322         nitrox_config_bmi_unit(ndev);
 323         nitrox_config_bmo_unit(ndev);
 324         /* configure Local Buffer Cache */
 325         nitrox_config_lbc_unit(ndev);
 326         nitrox_config_rand_unit(ndev);
 327
 328         /* load firmware on SE cores */
 329         err = nitrox_load_fw(ndev, SE_FW);
 330         if (err)
 331                 return err;
 332
 333         nitrox_config_emu_unit(ndev);
 334
 335         return 0;
 336 }
 337
 338 #if IS_ENABLED(CONFIG_DEBUG_FS)
 339 static int registers_show(struct seq_file *s, void *v)
 340 {
 341         struct nitrox_device *ndev = s->private;
 342         u64 offset;
 343
 344         /* NPS DMA stats */
 345         offset = NPS_STATS_PKT_DMA_RD_CNT;
 346         seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
 347                    nitrox_read_csr(ndev, offset));
 348         offset = NPS_STATS_PKT_DMA_WR_CNT;
 349         seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
 350                    nitrox_read_csr(ndev, offset));
 351
 352         /* BMI/BMO stats */
 353         offset = BMI_NPS_PKT_CNT;
 354         seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
 355                    nitrox_read_csr(ndev, offset));
 356         offset = BMO_NPS_SLC_PKT_CNT;
 357         seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
 358                    nitrox_read_csr(ndev, offset));
 359
 360         return 0;
 361 }
 362
 363 static int registers_open(struct inode *inode, struct file *file)
 364 {
 365         return single_open(file, registers_show, inode->i_private);
 366 }
 367
 368 static const struct file_operations register_fops = {
 369         .owner = THIS_MODULE,
 370         .open = registers_open,
 371         .read = seq_read,
 372         .llseek = seq_lseek,
 373         .release = single_release,
 374 };
 375
 376 static int firmware_show(struct seq_file *s, void *v)
 377 {
 378         struct nitrox_device *ndev = s->private;
 379
 380         seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
 381         return 0;
 382 }
 383
 384 static int firmware_open(struct inode *inode, struct file *file)
 385 {
 386         return single_open(file, firmware_show, inode->i_private);
 387 }
 388
 389 static const struct file_operations firmware_fops = {
 390         .owner = THIS_MODULE,
 391         .open = firmware_open,
 392         .read = seq_read,
 393         .llseek = seq_lseek,
 394         .release = single_release,
 395 };
 396
 397 static int nitrox_show(struct seq_file *s, void *v)
 398 {
 399         struct nitrox_device *ndev = s->private;
 400
 401         seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
 402         seq_printf(s, "  Revision ID: 0x%0x\n", ndev->hw.revision_id);
 403         seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
 404                    ndev->hw.ae_cores, ndev->hw.se_cores);
 405         seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
 406         seq_printf(s, "  Queue length: %u\n", ndev->qlen);
 407         seq_printf(s, "  Node: %u\n", ndev->node);
 408
 409         return 0;
 410 }
 411
 412 static int nitrox_open(struct inode *inode, struct file *file)
 413 {
 414         return single_open(file, nitrox_show, inode->i_private);
 415 }
 416
 417 static const struct file_operations nitrox_fops = {
 418         .owner = THIS_MODULE,
 419         .open = nitrox_open,
 420         .read = seq_read,
 421         .llseek = seq_lseek,
 422         .release = single_release,
 423 };
 424
 425 static void nitrox_debugfs_exit(struct nitrox_device *ndev)
 426 {
 427         debugfs_remove_recursive(ndev->debugfs_dir);
 428         ndev->debugfs_dir = NULL;
 429 }
 430
 431 static int nitrox_debugfs_init(struct nitrox_device *ndev)
 432 {
 433         struct dentry *dir, *f;
 434
 435         dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
 436         if (!dir)
 437                 return -ENOMEM;
 438
 439         ndev->debugfs_dir = dir;
 440         f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
 441         if (!f)
 442                 goto err;
 443         f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
 444         if (!f)
 445                 goto err;
 446         f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
 447         if (!f)
 448                 goto err;
 449
 450         return 0;
 451
 452 err:
 453         nitrox_debugfs_exit(ndev);
 454         return -ENODEV;
 455 }
 456 #else
 457 static int nitrox_debugfs_init(struct nitrox_device *ndev)
 458 {
 459         return 0;
 460 }
 461
 462 static void nitrox_debugfs_exit(struct nitrox_device *ndev)
 463 {
 464 }
 465 #endif
 466
 467 /**
 468  * nitrox_probe - NITROX Initialization function.
 469  * @pdev: PCI device information struct
 470  * @id: entry in nitrox_pci_tbl
 471  *
 472  * Return: 0, if the driver is bound to the device, or
 473  *         a negative error if there is failure.
 474  */
 475 static int nitrox_probe(struct pci_dev *pdev,
 476                         const struct pci_device_id *id)
 477 {
 478         struct nitrox_device *ndev;
 479         int err;
 480
 481         dev_info_once(&pdev->dev, "%s driver version %s\n",
 482                       nitrox_driver_name, DRIVER_VERSION);
 483
 484         err = pci_enable_device_mem(pdev);
 485         if (err)
 486                 return err;
 487
 488         /* do FLR */
 489         err = nitrox_reset_device(pdev);
 490         if (err) {
 491                 dev_err(&pdev->dev, "FLR failed\n");
 492                 pci_disable_device(pdev);
 493                 return err;
 494         }
 495
 496         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
 497                 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
 498         } else {
 499                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 500                 if (err) {
 501                         dev_err(&pdev->dev, "DMA configuration failed\n");
 502                         pci_disable_device(pdev);
 503                         return err;
 504                 }
 505         }
 506
 507         err = pci_request_mem_regions(pdev, nitrox_driver_name);
 508         if (err) {
 509                 pci_disable_device(pdev);
 510                 return err;
 511         }
 512         pci_set_master(pdev);
 513
 514         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 515         if (!ndev)
 516                 goto ndev_fail;
 517
 518         pci_set_drvdata(pdev, ndev);
 519         ndev->pdev = pdev;
 520
 521         /* add to device list */
 522         nitrox_add_to_devlist(ndev);
 523
 524         ndev->hw.vendor_id = pdev->vendor;
 525         ndev->hw.device_id = pdev->device;
 526         ndev->hw.revision_id = pdev->revision;
 527         /* command timeout in jiffies */
 528         ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
 529         ndev->node = dev_to_node(&pdev->dev);
 530         if (ndev->node == NUMA_NO_NODE)
 531                 ndev->node = 0;
 532
 533         ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
 534                                  pci_resource_len(pdev, 0));
 535         if (!ndev->bar_addr) {
 536                 err = -EIO;
 537                 goto ioremap_err;
 538         }
 539         /* allocate command queus based on cpus, max queues are 64 */
 540         ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
 541         ndev->qlen = qlen;
 542
 543         err = nitrox_pf_sw_init(ndev);
 544         if (err)
 545                 goto ioremap_err;
 546
 547         err = nitrox_pf_hw_init(ndev);
 548         if (err)
 549                 goto pf_hw_fail;
 550
 551         err = nitrox_debugfs_init(ndev);
 552         if (err)
 553                 goto pf_hw_fail;
 554
 555         set_bit(NITROX_READY, &ndev->status);
 556         /* barrier to sync with other cpus */
 557         smp_mb__after_atomic();
 558
 559         err = nitrox_crypto_register();
 560         if (err)
 561                 goto crypto_fail;
 562
 563         return 0;
 564
 565 crypto_fail:
 566         nitrox_debugfs_exit(ndev);
 567         clear_bit(NITROX_READY, &ndev->status);
 568         /* barrier to sync with other cpus */
 569         smp_mb__after_atomic();
 570 pf_hw_fail:
 571         nitrox_pf_sw_cleanup(ndev);
 572 ioremap_err:
 573         nitrox_remove_from_devlist(ndev);
 574         kfree(ndev);
 575         pci_set_drvdata(pdev, NULL);
 576 ndev_fail:
 577         pci_release_mem_regions(pdev);
 578         pci_disable_device(pdev);
 579         return err;
 580 }
 581
 582 /**
 583  * nitrox_remove - Unbind the driver from the device.
 584  * @pdev: PCI device information struct
 585  */
 586 static void nitrox_remove(struct pci_dev *pdev)
 587 {
 588         struct nitrox_device *ndev = pci_get_drvdata(pdev);
 589
 590         if (!ndev)
 591                 return;
 592
 593         if (!refcount_dec_and_test(&ndev->refcnt)) {
 594                 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
 595                         refcount_read(&ndev->refcnt));
 596                 return;
 597         }
 598
 599         dev_info(DEV(ndev), "Removing Device %x:%x\n",
 600                  ndev->hw.vendor_id, ndev->hw.device_id);
 601
 602         clear_bit(NITROX_READY, &ndev->status);
 603         /* barrier to sync with other cpus */
 604         smp_mb__after_atomic();
 605
 606         nitrox_remove_from_devlist(ndev);
 607         nitrox_crypto_unregister();
 608         nitrox_debugfs_exit(ndev);
 609         nitrox_pf_sw_cleanup(ndev);
 610
 611         iounmap(ndev->bar_addr);
 612         kfree(ndev);
 613
 614         pci_set_drvdata(pdev, NULL);
 615         pci_release_mem_regions(pdev);
 616         pci_disable_device(pdev);
 617 }
 618
 619 static void nitrox_shutdown(struct pci_dev *pdev)
 620 {
 621         pci_set_drvdata(pdev, NULL);
 622         pci_release_mem_regions(pdev);
 623         pci_disable_device(pdev);
 624 }
 625
 626 static struct pci_driver nitrox_driver = {
 627         .name = nitrox_driver_name,
 628         .id_table = nitrox_pci_tbl,
 629         .probe = nitrox_probe,
 630         .remove = nitrox_remove,
 631         .shutdown = nitrox_shutdown,
 632 };
 633
 634 module_pci_driver(nitrox_driver);
 635
 636 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
 637 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
 638 MODULE_LICENSE("GPL");
 639 MODULE_VERSION(DRIVER_VERSION);
 640 MODULE_FIRMWARE(SE_FW);