drivers/crypto/cavium/nitrox/nitrox_main.c

   1 #include <linux/aer.h>
   2 #include <linux/delay.h>
   3 #include <linux/debugfs.h>
   4 #include <linux/firmware.h>
   5 #include <linux/list.h>
   6 #include <linux/module.h>
   7 #include <linux/mutex.h>
   8 #include <linux/pci.h>
   9 #include <linux/pci_ids.h>
  10
  11 #include "nitrox_dev.h"
  12 #include "nitrox_common.h"
  13 #include "nitrox_csr.h"
  14
  15 #define CNN55XX_DEV_ID  0x12
  16 #define MAX_PF_QUEUES   64
  17 #define UCODE_HLEN 48
  18 #define SE_GROUP 0
  19
  20 #define DRIVER_VERSION "1.0"
  21 /* SE microcode */
  22 #define SE_FW   "cnn55xx_se.fw"
  23
  24 static const char nitrox_driver_name[] = "CNN55XX";
  25
  26 static LIST_HEAD(ndevlist);
  27 static DEFINE_MUTEX(devlist_lock);
  28 static unsigned int num_devices;
  29
  30 /**
  31  * nitrox_pci_tbl - PCI Device ID Table
  32  */
  33 static const struct pci_device_id nitrox_pci_tbl[] = {
  34         {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
  35         /* required last entry */
  36         {0, }
  37 };
  38 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
  39
  40 static unsigned int qlen = DEFAULT_CMD_QLEN;
  41 module_param(qlen, uint, 0644);
  42 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
  43
  44 /**
  45  * struct ucode - Firmware Header
  46  * @id: microcode ID
  47  * @version: firmware version
  48  * @code_size: code section size
  49  * @raz: alignment
  50  * @code: code section
  51  */
  52 struct ucode {
  53         u8 id;
  54         char version[VERSION_LEN - 1];
  55         __be32 code_size;
  56         u8 raz[12];
  57         u64 code[0];
  58 };
  59
  60 /**
  61  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
  62  */
  63 static void write_to_ucd_unit(struct nitrox_device *ndev,
  64                               struct ucode *ucode)
  65 {
  66         u32 code_size = be32_to_cpu(ucode->code_size) * 2;
  67         u64 offset, data;
  68         int i = 0;
  69
  70         /*
  71          * UCD structure
  72          *
  73          *  -------------
  74          *  |    BLK 7  |
  75          *  -------------
  76          *  |    BLK 6  |
  77          *  -------------
  78          *  |    ...    |
  79          *  -------------
  80          *  |    BLK 0  |
  81          *  -------------
  82          *  Total of 8 blocks, each size 32KB
  83          */
  84
  85         /* set the block number */
  86         offset = UCD_UCODE_LOAD_BLOCK_NUM;
  87         nitrox_write_csr(ndev, offset, 0);
  88
  89         code_size = roundup(code_size, 8);
  90         while (code_size) {
  91                 data = ucode->code[i];
  92                 /* write 8 bytes at a time */
  93                 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
  94                 nitrox_write_csr(ndev, offset, data);
  95                 code_size -= 8;
  96                 i++;
  97         }
  98
  99         /* put all SE cores in group 0 */
 100         offset = POM_GRP_EXECMASKX(SE_GROUP);
 101         nitrox_write_csr(ndev, offset, (~0ULL));
 102
 103         for (i = 0; i < ndev->hw.se_cores; i++) {
 104                 /*
 105                  * write block number and firware length
 106                  * bit:<2:0> block number
 107                  * bit:3 is set SE uses 32KB microcode
 108                  * bit:3 is clear SE uses 64KB microcode
 109                  */
 110                 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
 111                 nitrox_write_csr(ndev, offset, 0x8);
 112         }
 113         usleep_range(300, 400);
 114 }
 115
 116 static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
 117 {
 118         const struct firmware *fw;
 119         struct ucode *ucode;
 120         int ret;
 121
 122         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 123
 124         ret = request_firmware(&fw, fw_name, DEV(ndev));
 125         if (ret < 0) {
 126                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 127                 return ret;
 128         }
 129
 130         ucode = (struct ucode *)fw->data;
 131         /* copy the firmware version */
 132         memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
 133         ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
 134
 135         write_to_ucd_unit(ndev, ucode);
 136         release_firmware(fw);
 137
 138         set_bit(NITROX_UCODE_LOADED, &ndev->status);
 139         /* barrier to sync with other cpus */
 140         smp_mb__after_atomic();
 141         return 0;
 142 }
 143
 144 /**
 145  * nitrox_add_to_devlist - add NITROX device to global device list
 146  * @ndev: NITROX device
 147  */
 148 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
 149 {
 150         struct nitrox_device *dev;
 151         int ret = 0;
 152
 153         INIT_LIST_HEAD(&ndev->list);
 154         refcount_set(&ndev->refcnt, 1);
 155
 156         mutex_lock(&devlist_lock);
 157         list_for_each_entry(dev, &ndevlist, list) {
 158                 if (dev == ndev) {
 159                         ret = -EEXIST;
 160                         goto unlock;
 161                 }
 162         }
 163         ndev->idx = num_devices++;
 164         list_add_tail(&ndev->list, &ndevlist);
 165 unlock:
 166         mutex_unlock(&devlist_lock);
 167         return ret;
 168 }
 169
 170 /**
 171  * nitrox_remove_from_devlist - remove NITROX device from
 172  *   global device list
 173  * @ndev: NITROX device
 174  */
 175 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
 176 {
 177         mutex_lock(&devlist_lock);
 178         list_del(&ndev->list);
 179         num_devices--;
 180         mutex_unlock(&devlist_lock);
 181 }
 182
 183 static int nitrox_reset_device(struct pci_dev *pdev)
 184 {
 185         int pos = 0;
 186
 187         pos = pci_save_state(pdev);
 188         if (pos) {
 189                 dev_err(&pdev->dev, "Failed to save pci state\n");
 190                 return -ENOMEM;
 191         }
 192
 193         pos = pci_pcie_cap(pdev);
 194         if (!pos)
 195                 return -ENOTTY;
 196
 197         if (!pci_wait_for_pending_transaction(pdev))
 198                 dev_err(&pdev->dev, "waiting for pending transaction\n");
 199
 200         pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
 201         msleep(100);
 202         pci_restore_state(pdev);
 203
 204         return 0;
 205 }
 206
 207 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
 208 {
 209         int err;
 210
 211         err = nitrox_common_sw_init(ndev);
 212         if (err)
 213                 return err;
 214
 215         err = nitrox_pf_init_isr(ndev);
 216         if (err)
 217                 nitrox_common_sw_cleanup(ndev);
 218
 219         return err;
 220 }
 221
 222 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
 223 {
 224         nitrox_pf_cleanup_isr(ndev);
 225         nitrox_common_sw_cleanup(ndev);
 226 }
 227
 228 /**
 229  * nitrox_bist_check - Check NITORX BIST registers status
 230  * @ndev: NITROX device
 231  */
 232 static int nitrox_bist_check(struct nitrox_device *ndev)
 233 {
 234         u64 value = 0;
 235         int i;
 236
 237         for (i = 0; i < NR_CLUSTERS; i++) {
 238                 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
 239                 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
 240         }
 241         value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
 242         value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
 243         value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
 244         value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
 245         value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
 246         value += nitrox_read_csr(ndev, POM_BIST_REG);
 247         value += nitrox_read_csr(ndev, BMI_BIST_REG);
 248         value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
 249         value += nitrox_read_csr(ndev, BMO_BIST_REG);
 250         value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
 251         value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
 252         if (value)
 253                 return -EIO;
 254         return 0;
 255 }
 256
 257 static void nitrox_get_hwinfo(struct nitrox_device *ndev)
 258 {
 259         union emu_fuse_map emu_fuse;
 260         u64 offset;
 261         int i;
 262
 263         for (i = 0; i < NR_CLUSTERS; i++) {
 264                 u8 dead_cores;
 265
 266                 offset = EMU_FUSE_MAPX(i);
 267                 emu_fuse.value = nitrox_read_csr(ndev, offset);
 268                 if (emu_fuse.s.valid) {
 269                         dead_cores = hweight32(emu_fuse.s.ae_fuse);
 270                         ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
 271                         dead_cores = hweight16(emu_fuse.s.se_fuse);
 272                         ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
 273                 }
 274         }
 275 }
 276
 277 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
 278 {
 279         int err;
 280
 281         err = nitrox_bist_check(ndev);
 282         if (err) {
 283                 dev_err(&ndev->pdev->dev, "BIST check failed\n");
 284                 return err;
 285         }
 286         /* get cores information */
 287         nitrox_get_hwinfo(ndev);
 288
 289         nitrox_config_nps_unit(ndev);
 290         nitrox_config_pom_unit(ndev);
 291         nitrox_config_efl_unit(ndev);
 292         /* configure IO units */
 293         nitrox_config_bmi_unit(ndev);
 294         nitrox_config_bmo_unit(ndev);
 295         /* configure Local Buffer Cache */
 296         nitrox_config_lbc_unit(ndev);
 297         nitrox_config_rand_unit(ndev);
 298
 299         /* load firmware on SE cores */
 300         err = nitrox_load_fw(ndev, SE_FW);
 301         if (err)
 302                 return err;
 303
 304         nitrox_config_emu_unit(ndev);
 305
 306         return 0;
 307 }
 308
 309 #if IS_ENABLED(CONFIG_DEBUG_FS)
 310 static int registers_show(struct seq_file *s, void *v)
 311 {
 312         struct nitrox_device *ndev = s->private;
 313         u64 offset;
 314
 315         /* NPS DMA stats */
 316         offset = NPS_STATS_PKT_DMA_RD_CNT;
 317         seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
 318                    nitrox_read_csr(ndev, offset));
 319         offset = NPS_STATS_PKT_DMA_WR_CNT;
 320         seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
 321                    nitrox_read_csr(ndev, offset));
 322
 323         /* BMI/BMO stats */
 324         offset = BMI_NPS_PKT_CNT;
 325         seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
 326                    nitrox_read_csr(ndev, offset));
 327         offset = BMO_NPS_SLC_PKT_CNT;
 328         seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
 329                    nitrox_read_csr(ndev, offset));
 330
 331         return 0;
 332 }
 333
 334 static int registers_open(struct inode *inode, struct file *file)
 335 {
 336         return single_open(file, registers_show, inode->i_private);
 337 }
 338
 339 static const struct file_operations register_fops = {
 340         .owner = THIS_MODULE,
 341         .open = registers_open,
 342         .read = seq_read,
 343         .llseek = seq_lseek,
 344         .release = single_release,
 345 };
 346
 347 static int firmware_show(struct seq_file *s, void *v)
 348 {
 349         struct nitrox_device *ndev = s->private;
 350
 351         seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
 352         return 0;
 353 }
 354
 355 static int firmware_open(struct inode *inode, struct file *file)
 356 {
 357         return single_open(file, firmware_show, inode->i_private);
 358 }
 359
 360 static const struct file_operations firmware_fops = {
 361         .owner = THIS_MODULE,
 362         .open = firmware_open,
 363         .read = seq_read,
 364         .llseek = seq_lseek,
 365         .release = single_release,
 366 };
 367
 368 static int nitrox_show(struct seq_file *s, void *v)
 369 {
 370         struct nitrox_device *ndev = s->private;
 371
 372         seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
 373         seq_printf(s, "  Revsion ID: 0x%0x\n", ndev->hw.revision_id);
 374         seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
 375                    ndev->hw.ae_cores, ndev->hw.se_cores);
 376         seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
 377         seq_printf(s, "  Queue length: %u\n", ndev->qlen);
 378         seq_printf(s, "  Node: %u\n", ndev->node);
 379
 380         return 0;
 381 }
 382
 383 static int nitrox_open(struct inode *inode, struct file *file)
 384 {
 385         return single_open(file, nitrox_show, inode->i_private);
 386 }
 387
 388 static const struct file_operations nitrox_fops = {
 389         .owner = THIS_MODULE,
 390         .open = nitrox_open,
 391         .read = seq_read,
 392         .llseek = seq_lseek,
 393         .release = single_release,
 394 };
 395
 396 static void nitrox_debugfs_exit(struct nitrox_device *ndev)
 397 {
 398         debugfs_remove_recursive(ndev->debugfs_dir);
 399         ndev->debugfs_dir = NULL;
 400 }
 401
 402 static int nitrox_debugfs_init(struct nitrox_device *ndev)
 403 {
 404         struct dentry *dir, *f;
 405
 406         dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
 407         if (!dir)
 408                 return -ENOMEM;
 409
 410         ndev->debugfs_dir = dir;
 411         f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
 412         if (!f)
 413                 goto err;
 414         f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
 415         if (!f)
 416                 goto err;
 417         f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
 418         if (!f)
 419                 goto err;
 420
 421         return 0;
 422
 423 err:
 424         nitrox_debugfs_exit(ndev);
 425         return -ENODEV;
 426 }
 427 #else
 428 static int nitrox_debugfs_init(struct nitrox_device *ndev)
 429 {
 430         return 0;
 431 }
 432
 433 static void nitrox_debugfs_exit(struct nitrox_device *ndev)
 434 {
 435 }
 436 #endif
 437
 438 /**
 439  * nitrox_probe - NITROX Initialization function.
 440  * @pdev: PCI device information struct
 441  * @id: entry in nitrox_pci_tbl
 442  *
 443  * Return: 0, if the driver is bound to the device, or
 444  *         a negative error if there is failure.
 445  */
 446 static int nitrox_probe(struct pci_dev *pdev,
 447                         const struct pci_device_id *id)
 448 {
 449         struct nitrox_device *ndev;
 450         int err;
 451
 452         dev_info_once(&pdev->dev, "%s driver version %s\n",
 453                       nitrox_driver_name, DRIVER_VERSION);
 454
 455         err = pci_enable_device_mem(pdev);
 456         if (err)
 457                 return err;
 458
 459         /* do FLR */
 460         err = nitrox_reset_device(pdev);
 461         if (err) {
 462                 dev_err(&pdev->dev, "FLR failed\n");
 463                 pci_disable_device(pdev);
 464                 return err;
 465         }
 466
 467         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
 468                 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
 469         } else {
 470                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 471                 if (err) {
 472                         dev_err(&pdev->dev, "DMA configuration failed\n");
 473                         pci_disable_device(pdev);
 474                         return err;
 475                 }
 476         }
 477
 478         err = pci_request_mem_regions(pdev, nitrox_driver_name);
 479         if (err) {
 480                 pci_disable_device(pdev);
 481                 return err;
 482         }
 483         pci_set_master(pdev);
 484
 485         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 486         if (!ndev)
 487                 goto ndev_fail;
 488
 489         pci_set_drvdata(pdev, ndev);
 490         ndev->pdev = pdev;
 491
 492         /* add to device list */
 493         nitrox_add_to_devlist(ndev);
 494
 495         ndev->hw.vendor_id = pdev->vendor;
 496         ndev->hw.device_id = pdev->device;
 497         ndev->hw.revision_id = pdev->revision;
 498         /* command timeout in jiffies */
 499         ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
 500         ndev->node = dev_to_node(&pdev->dev);
 501         if (ndev->node == NUMA_NO_NODE)
 502                 ndev->node = 0;
 503
 504         ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
 505                                  pci_resource_len(pdev, 0));
 506         if (!ndev->bar_addr) {
 507                 err = -EIO;
 508                 goto ioremap_err;
 509         }
 510         /* allocate command queus based on cpus, max queues are 64 */
 511         ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
 512         ndev->qlen = qlen;
 513
 514         err = nitrox_pf_sw_init(ndev);
 515         if (err)
 516                 goto ioremap_err;
 517
 518         err = nitrox_pf_hw_init(ndev);
 519         if (err)
 520                 goto pf_hw_fail;
 521
 522         err = nitrox_debugfs_init(ndev);
 523         if (err)
 524                 goto pf_hw_fail;
 525
 526         set_bit(NITROX_READY, &ndev->status);
 527         /* barrier to sync with other cpus */
 528         smp_mb__after_atomic();
 529         return 0;
 530
 531 pf_hw_fail:
 532         nitrox_pf_sw_cleanup(ndev);
 533 ioremap_err:
 534         nitrox_remove_from_devlist(ndev);
 535         kfree(ndev);
 536         pci_set_drvdata(pdev, NULL);
 537 ndev_fail:
 538         pci_release_mem_regions(pdev);
 539         pci_disable_device(pdev);
 540         return err;
 541 }
 542
 543 /**
 544  * nitrox_remove - Unbind the driver from the device.
 545  * @pdev: PCI device information struct
 546  */
 547 static void nitrox_remove(struct pci_dev *pdev)
 548 {
 549         struct nitrox_device *ndev = pci_get_drvdata(pdev);
 550
 551         if (!ndev)
 552                 return;
 553
 554         if (!refcount_dec_and_test(&ndev->refcnt)) {
 555                 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
 556                         refcount_read(&ndev->refcnt));
 557                 return;
 558         }
 559
 560         dev_info(DEV(ndev), "Removing Device %x:%x\n",
 561                  ndev->hw.vendor_id, ndev->hw.device_id);
 562
 563         clear_bit(NITROX_READY, &ndev->status);
 564         /* barrier to sync with other cpus */
 565         smp_mb__after_atomic();
 566
 567         nitrox_remove_from_devlist(ndev);
 568         nitrox_debugfs_exit(ndev);
 569         nitrox_pf_sw_cleanup(ndev);
 570
 571         iounmap(ndev->bar_addr);
 572         kfree(ndev);
 573
 574         pci_set_drvdata(pdev, NULL);
 575         pci_release_mem_regions(pdev);
 576         pci_disable_device(pdev);
 577 }
 578
 579 static void nitrox_shutdown(struct pci_dev *pdev)
 580 {
 581         pci_set_drvdata(pdev, NULL);
 582         pci_release_mem_regions(pdev);
 583         pci_disable_device(pdev);
 584 }
 585
 586 static struct pci_driver nitrox_driver = {
 587         .name = nitrox_driver_name,
 588         .id_table = nitrox_pci_tbl,
 589         .probe = nitrox_probe,
 590         .remove = nitrox_remove,
 591         .shutdown = nitrox_shutdown,
 592 };
 593
 594 module_pci_driver(nitrox_driver);
 595
 596 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
 597 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
 598 MODULE_LICENSE("GPL");
 599 MODULE_VERSION(DRIVER_VERSION);
 600 MODULE_FIRMWARE(SE_FW);