drivers/crypto/qat/qat_dh895xcc/adf_drv.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2014 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2014 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/kernel.h>
  48 #include <linux/module.h>
  49 #include <linux/pci.h>
  50 #include <linux/init.h>
  51 #include <linux/types.h>
  52 #include <linux/fs.h>
  53 #include <linux/slab.h>
  54 #include <linux/errno.h>
  55 #include <linux/device.h>
  56 #include <linux/dma-mapping.h>
  57 #include <linux/platform_device.h>
  58 #include <linux/workqueue.h>
  59 #include <linux/io.h>
  60 #include <adf_accel_devices.h>
  61 #include <adf_common_drv.h>
  62 #include <adf_cfg.h>
  63 #include <adf_transport_access_macros.h>
  64 #include "adf_dh895xcc_hw_data.h"
  65 #include "adf_drv.h"
  66
  67 static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;
  68
  69 #define ADF_SYSTEM_DEVICE(device_id) \
  70         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  71
  72 static const struct pci_device_id adf_pci_tbl[] = {
  73         ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
  74         {0,}
  75 };
  76 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
  77
  78 static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
  79 static void adf_remove(struct pci_dev *dev);
  80
  81 static struct pci_driver adf_driver = {
  82         .id_table = adf_pci_tbl,
  83         .name = adf_driver_name,
  84         .probe = adf_probe,
  85         .remove = adf_remove
  86 };
  87
  88 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
  89 {
  90         struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
  91         int i;
  92
  93         adf_dev_shutdown(accel_dev);
  94
  95         for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
  96                 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
  97
  98                 if (bar->virt_addr)
  99                         pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
 100         }
 101
 102         if (accel_dev->hw_device) {
 103                 switch (accel_dev->hw_device->pci_dev_id) {
 104                 case ADF_DH895XCC_PCI_DEVICE_ID:
 105                         adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
 106                         break;
 107                 default:
 108                         break;
 109                 }
 110                 kfree(accel_dev->hw_device);
 111         }
 112         adf_cfg_dev_remove(accel_dev);
 113         debugfs_remove(accel_dev->debugfs_dir);
 114         adf_devmgr_rm_dev(accel_dev);
 115         pci_release_regions(accel_pci_dev->pci_dev);
 116         pci_disable_device(accel_pci_dev->pci_dev);
 117         kfree(accel_dev);
 118 }
 119
 120 static int adf_dev_configure(struct adf_accel_dev *accel_dev)
 121 {
 122         int cpus = num_online_cpus();
 123         int banks = GET_MAX_BANKS(accel_dev);
 124         int instances = min(cpus, banks);
 125         char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
 126         int i;
 127         unsigned long val;
 128
 129         if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
 130                 goto err;
 131         if (adf_cfg_section_add(accel_dev, "Accelerator0"))
 132                 goto err;
 133         for (i = 0; i < instances; i++) {
 134                 val = i;
 135                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
 136                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 137                                                 key, (void *)&val, ADF_DEC))
 138                         goto err;
 139
 140                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
 141                          i);
 142                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 143                                                 key, (void *)&val, ADF_DEC))
 144                         goto err;
 145
 146                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
 147                 val = 128;
 148                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 149                                                 key, (void *)&val, ADF_DEC))
 150                         goto err;
 151
 152                 val = 512;
 153                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
 154                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 155                                                 key, (void *)&val, ADF_DEC))
 156                         goto err;
 157
 158                 val = 0;
 159                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
 160                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 161                                                 key, (void *)&val, ADF_DEC))
 162                         goto err;
 163
 164                 val = 2;
 165                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
 166                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 167                                                 key, (void *)&val, ADF_DEC))
 168                         goto err;
 169
 170                 val = 4;
 171                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i);
 172                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 173                                                 key, (void *)&val, ADF_DEC))
 174                         goto err;
 175
 176                 val = 8;
 177                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
 178                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 179                                                 key, (void *)&val, ADF_DEC))
 180                         goto err;
 181
 182                 val = 10;
 183                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
 184                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 185                                                 key, (void *)&val, ADF_DEC))
 186                         goto err;
 187
 188                 val = 12;
 189                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i);
 190                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 191                                                 key, (void *)&val, ADF_DEC))
 192                         goto err;
 193
 194                 val = ADF_COALESCING_DEF_TIME;
 195                 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
 196                 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
 197                                                 key, (void *)&val, ADF_DEC))
 198                         goto err;
 199         }
 200
 201         val = i;
 202         if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 203                                         ADF_NUM_CY, (void *)&val, ADF_DEC))
 204                 goto err;
 205
 206         set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
 207         return 0;
 208 err:
 209         dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
 210         return -EINVAL;
 211 }
 212
 213 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 214 {
 215         struct adf_accel_dev *accel_dev;
 216         struct adf_accel_pci *accel_pci_dev;
 217         struct adf_hw_device_data *hw_data;
 218         char name[ADF_DEVICE_NAME_LENGTH];
 219         unsigned int i, bar_nr;
 220         int ret;
 221
 222         switch (ent->device) {
 223         case ADF_DH895XCC_PCI_DEVICE_ID:
 224                 break;
 225         default:
 226                 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
 227                 return -ENODEV;
 228         }
 229
 230         if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
 231                 /* If the accelerator is connected to a node with no memory
 232                  * there is no point in using the accelerator since the remote
 233                  * memory transaction will be very slow. */
 234                 dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
 235                 return -EINVAL;
 236         }
 237
 238         accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
 239                                  dev_to_node(&pdev->dev));
 240         if (!accel_dev)
 241                 return -ENOMEM;
 242
 243         INIT_LIST_HEAD(&accel_dev->crypto_list);
 244
 245         /* Add accel device to accel table.
 246          * This should be called before adf_cleanup_accel is called */
 247         if (adf_devmgr_add_dev(accel_dev)) {
 248                 dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
 249                 kfree(accel_dev);
 250                 return -EFAULT;
 251         }
 252
 253         accel_dev->owner = THIS_MODULE;
 254         /* Allocate and configure device configuration structure */
 255         hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
 256                                dev_to_node(&pdev->dev));
 257         if (!hw_data) {
 258                 ret = -ENOMEM;
 259                 goto out_err;
 260         }
 261
 262         accel_dev->hw_device = hw_data;
 263         switch (ent->device) {
 264         case ADF_DH895XCC_PCI_DEVICE_ID:
 265                 adf_init_hw_data_dh895xcc(accel_dev->hw_device);
 266                 break;
 267         default:
 268                 return -ENODEV;
 269         }
 270         accel_pci_dev = &accel_dev->accel_pci_dev;
 271         pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
 272         pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
 273                               &hw_data->fuses);
 274
 275         /* Get Accelerators and Accelerators Engines masks */
 276         hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
 277         hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
 278         accel_pci_dev->sku = hw_data->get_sku(hw_data);
 279         accel_pci_dev->pci_dev = pdev;
 280         /* If the device has no acceleration engines then ignore it. */
 281         if (!hw_data->accel_mask || !hw_data->ae_mask ||
 282             ((~hw_data->ae_mask) & 0x01)) {
 283                 dev_err(&pdev->dev, "No acceleration units found");
 284                 ret = -EFAULT;
 285                 goto out_err;
 286         }
 287
 288         /* Create dev top level debugfs entry */
 289         snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX,
 290                  hw_data->dev_class->name, hw_data->instance_id);
 291         accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
 292         if (!accel_dev->debugfs_dir) {
 293                 dev_err(&pdev->dev, "Could not create debugfs dir\n");
 294                 ret = -EINVAL;
 295                 goto out_err;
 296         }
 297
 298         /* Create device configuration table */
 299         ret = adf_cfg_dev_add(accel_dev);
 300         if (ret)
 301                 goto out_err;
 302
 303         /* enable PCI device */
 304         if (pci_enable_device(pdev)) {
 305                 ret = -EFAULT;
 306                 goto out_err;
 307         }
 308
 309         /* set dma identifier */
 310         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
 311                 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
 312                         dev_err(&pdev->dev, "No usable DMA configuration\n");
 313                         ret = -EFAULT;
 314                         goto out_err;
 315                 } else {
 316                         pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
 317                 }
 318
 319         } else {
 320                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
 321         }
 322
 323         if (pci_request_regions(pdev, adf_driver_name)) {
 324                 ret = -EFAULT;
 325                 goto out_err;
 326         }
 327
 328         /* Read accelerator capabilities mask */
 329         pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
 330                               &hw_data->accel_capabilities_mask);
 331
 332         /* Find and map all the device's BARS */
 333         for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
 334                 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
 335
 336                 bar_nr = i * 2;
 337                 bar->base_addr = pci_resource_start(pdev, bar_nr);
 338                 if (!bar->base_addr)
 339                         break;
 340                 bar->size = pci_resource_len(pdev, bar_nr);
 341                 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
 342                 if (!bar->virt_addr) {
 343                         dev_err(&pdev->dev, "Failed to map BAR %d\n", i);
 344                         ret = -EFAULT;
 345                         goto out_err;
 346                 }
 347         }
 348         pci_set_master(pdev);
 349
 350         if (adf_enable_aer(accel_dev, &adf_driver)) {
 351                 dev_err(&pdev->dev, "Failed to enable aer\n");
 352                 ret = -EFAULT;
 353                 goto out_err;
 354         }
 355
 356         if (pci_save_state(pdev)) {
 357                 dev_err(&pdev->dev, "Failed to save pci state\n");
 358                 ret = -ENOMEM;
 359                 goto out_err;
 360         }
 361
 362         ret = adf_dev_configure(accel_dev);
 363         if (ret)
 364                 goto out_err;
 365
 366         ret = adf_dev_init(accel_dev);
 367         if (ret)
 368                 goto out_err;
 369
 370         ret = adf_dev_start(accel_dev);
 371         if (ret) {
 372                 adf_dev_stop(accel_dev);
 373                 goto out_err;
 374         }
 375
 376         return 0;
 377 out_err:
 378         adf_cleanup_accel(accel_dev);
 379         return ret;
 380 }
 381
 382 static void adf_remove(struct pci_dev *pdev)
 383 {
 384         struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
 385
 386         if (!accel_dev) {
 387                 pr_err("QAT: Driver removal failed\n");
 388                 return;
 389         }
 390         if (adf_dev_stop(accel_dev))
 391                 dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
 392         adf_disable_aer(accel_dev);
 393         adf_cleanup_accel(accel_dev);
 394 }
 395
 396 static int __init adfdrv_init(void)
 397 {
 398         request_module("intel_qat");
 399         if (qat_admin_register())
 400                 return -EFAULT;
 401
 402         if (pci_register_driver(&adf_driver)) {
 403                 pr_err("QAT: Driver initialization failed\n");
 404                 return -EFAULT;
 405         }
 406         return 0;
 407 }
 408
 409 static void __exit adfdrv_release(void)
 410 {
 411         pci_unregister_driver(&adf_driver);
 412         qat_admin_unregister();
 413 }
 414
 415 module_init(adfdrv_init);
 416 module_exit(adfdrv_release);
 417
 418 MODULE_LICENSE("Dual BSD/GPL");
 419 MODULE_AUTHOR("Intel");
 420 MODULE_FIRMWARE("qat_895xcc.bin");
 421 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");