drivers/crypto/qat/qat_common/adf_sriov.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2015 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2015 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/workqueue.h>
  48 #include <linux/pci.h>
  49 #include <linux/device.h>
  50 #include <linux/iommu.h>
  51 #include "adf_common_drv.h"
  52 #include "adf_cfg.h"
  53 #include "adf_pf2vf_msg.h"
  54
  55 static struct workqueue_struct *pf2vf_resp_wq;
  56
  57 #define ME2FUNCTION_MAP_A_OFFSET        (0x3A400 + 0x190)
  58 #define ME2FUNCTION_MAP_A_NUM_REGS      96
  59
  60 #define ME2FUNCTION_MAP_B_OFFSET        (0x3A400 + 0x310)
  61 #define ME2FUNCTION_MAP_B_NUM_REGS      12
  62
  63 #define ME2FUNCTION_MAP_REG_SIZE        4
  64 #define ME2FUNCTION_MAP_VALID           BIT(7)
  65
  66 #define READ_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index)               \
  67         ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET +           \
  68                    ME2FUNCTION_MAP_REG_SIZE * index)
  69
  70 #define WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index, value)       \
  71         ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET +           \
  72                    ME2FUNCTION_MAP_REG_SIZE * index, value)
  73
  74 #define READ_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index)               \
  75         ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET +           \
  76                    ME2FUNCTION_MAP_REG_SIZE * index)
  77
  78 #define WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index, value)       \
  79         ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET +           \
  80                    ME2FUNCTION_MAP_REG_SIZE * index, value)
  81
  82 struct adf_pf2vf_resp_data {
  83         struct work_struct pf2vf_resp_work;
  84         struct adf_accel_dev *accel_dev;
  85         u32 resp;
  86         u8 vf_nr;
  87 };
  88
  89 static void adf_iov_send_resp(struct work_struct *work)
  90 {
  91         struct adf_pf2vf_resp_data *pf2vf_resp_data =
  92                 container_of(work, struct adf_pf2vf_resp_data, pf2vf_resp_work);
  93
  94         if (adf_iov_putmsg(pf2vf_resp_data->accel_dev, pf2vf_resp_data->resp,
  95                            pf2vf_resp_data->vf_nr)) {
  96                 dev_err(&GET_DEV(pf2vf_resp_data->accel_dev),
  97                         "Failed to send response\n");
  98         }
  99
 100         kfree(pf2vf_resp_data);
 101 }
 102
 103 static void adf_vf2pf_bh_handler(void *data)
 104 {
 105         struct adf_accel_vf_info *vf_info = (struct adf_accel_vf_info *)data;
 106         struct adf_accel_dev *accel_dev = vf_info->accel_dev;
 107         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 108         struct adf_bar *pmisc =
 109                         &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
 110         void __iomem *pmisc_addr = pmisc->virt_addr;
 111         u32 msg;
 112
 113         /* Read message from the VF */
 114         msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_info->vf_nr));
 115
 116         if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM))
 117                 /* Ignore legacy non-system (non-kernel) VF2PF messages */
 118                 goto err;
 119
 120         switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) {
 121         case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ:
 122                 {
 123                 u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT;
 124                 struct adf_pf2vf_resp_data *pf2vf_resp_data;
 125                 u32 resp = (ADF_PF2VF_MSGORIGIN_SYSTEM |
 126                                  (ADF_PF2VF_MSGTYPE_VERSION_RESP <<
 127                                   ADF_PF2VF_MSGTYPE_SHIFT) |
 128                                  (ADF_PFVF_COMPATIBILITY_VERSION <<
 129                                   ADF_PF2VF_VERSION_RESP_VERS_SHIFT));
 130
 131                 dev_dbg(&GET_DEV(accel_dev),
 132                         "Compatibility Version Request from VF%d vers=%u\n",
 133                         vf_info->vf_nr + 1, vf_compat_ver);
 134
 135                 if (vf_compat_ver < hw_data->min_iov_compat_ver) {
 136                         dev_err(&GET_DEV(accel_dev),
 137                                 "VF (vers %d) incompatible with PF (vers %d)\n",
 138                                 vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION);
 139                         resp |= ADF_PF2VF_VF_INCOMPATIBLE <<
 140                                 ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
 141                 } else if (vf_compat_ver > ADF_PFVF_COMPATIBILITY_VERSION) {
 142                         dev_err(&GET_DEV(accel_dev),
 143                                 "VF (vers %d) compat with PF (vers %d) unkn.\n",
 144                                 vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION);
 145                         resp |= ADF_PF2VF_VF_COMPAT_UNKNOWN <<
 146                                 ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
 147                 } else {
 148                         dev_dbg(&GET_DEV(accel_dev),
 149                                 "VF (vers %d) compatible with PF (vers %d)\n",
 150                                 vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION);
 151                         resp |= ADF_PF2VF_VF_COMPATIBLE <<
 152                                 ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
 153                 }
 154
 155                 pf2vf_resp_data = kzalloc(sizeof(*pf2vf_resp_data), GFP_ATOMIC);
 156                 if (!pf2vf_resp_data)
 157                         return;
 158
 159                 pf2vf_resp_data->accel_dev = accel_dev;
 160                 pf2vf_resp_data->vf_nr = vf_info->vf_nr;
 161                 pf2vf_resp_data->resp = resp;
 162                 INIT_WORK(&pf2vf_resp_data->pf2vf_resp_work, adf_iov_send_resp);
 163                 queue_work(pf2vf_resp_wq, &pf2vf_resp_data->pf2vf_resp_work);
 164                 }
 165                 break;
 166         case ADF_VF2PF_MSGTYPE_INIT:
 167                 {
 168                 dev_dbg(&GET_DEV(accel_dev),
 169                         "Init message received from VF%d 0x%x\n",
 170                         vf_info->vf_nr + 1, msg);
 171                 vf_info->init = true;
 172                 }
 173                 break;
 174         case ADF_VF2PF_MSGTYPE_SHUTDOWN:
 175                 {
 176                 dev_dbg(&GET_DEV(accel_dev),
 177                         "Shutdown message received from VF%d 0x%x\n",
 178                         vf_info->vf_nr + 1, msg);
 179                 vf_info->init = false;
 180                 }
 181                 break;
 182         case ADF_VF2PF_MSGTYPE_VERSION_REQ:
 183                 dev_err(&GET_DEV(accel_dev),
 184                         "Incompatible VersionRequest received from VF%d 0x%x\n",
 185                         vf_info->vf_nr + 1, msg);
 186                 break;
 187         default:
 188                 goto err;
 189         }
 190
 191         /* To ACK, clear the VF2PFINT bit */
 192         msg &= ~ADF_VF2PF_INT;
 193         ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_info->vf_nr), msg);
 194
 195         /* re-enable interrupt on PF from this VF */
 196         adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_info->vf_nr));
 197         return;
 198 err:
 199         dev_err(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x);\n",
 200                 vf_info->vf_nr + 1, msg);
 201 }
 202
 203 static int adf_enable_sriov(struct adf_accel_dev *accel_dev)
 204 {
 205         struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
 206         int totalvfs = pci_sriov_get_totalvfs(pdev);
 207         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 208         struct adf_bar *pmisc =
 209                         &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
 210         void __iomem *pmisc_addr = pmisc->virt_addr;
 211         struct adf_accel_vf_info *vf_info;
 212         int i;
 213         u32 reg;
 214
 215         /* Workqueue for PF2VF responses */
 216         pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq");
 217         if (!pf2vf_resp_wq)
 218                 return -ENOMEM;
 219
 220         for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs;
 221              i++, vf_info++) {
 222                 /* This ptr will be populated when VFs will be created */
 223                 vf_info->accel_dev = accel_dev;
 224                 vf_info->vf_nr = i;
 225
 226                 tasklet_init(&vf_info->vf2pf_bh_tasklet,
 227                              (void *)adf_vf2pf_bh_handler,
 228                              (unsigned long)vf_info);
 229                 mutex_init(&vf_info->pf2vf_lock);
 230                 ratelimit_state_init(&vf_info->vf2pf_ratelimit,
 231                                      DEFAULT_RATELIMIT_INTERVAL,
 232                                      DEFAULT_RATELIMIT_BURST);
 233         }
 234
 235         /* Set Valid bits in ME Thread to PCIe Function Mapping Group A */
 236         for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) {
 237                 reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i);
 238                 reg |= ME2FUNCTION_MAP_VALID;
 239                 WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg);
 240         }
 241
 242         /* Set Valid bits in ME Thread to PCIe Function Mapping Group B */
 243         for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) {
 244                 reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i);
 245                 reg |= ME2FUNCTION_MAP_VALID;
 246                 WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg);
 247         }
 248
 249         /* Enable VF to PF interrupts for all VFs */
 250         adf_enable_vf2pf_interrupts(accel_dev, GENMASK_ULL(totalvfs - 1, 0));
 251
 252         /*
 253          * Due to the hardware design, when SR-IOV and the ring arbiter
 254          * are enabled all the VFs supported in hardware must be enabled in
 255          * order for all the hardware resources (i.e. bundles) to be usable.
 256          * When SR-IOV is enabled, each of the VFs will own one bundle.
 257          */
 258         return pci_enable_sriov(pdev, totalvfs);
 259 }
 260
 261 /**
 262  * adf_disable_sriov() - Disable SRIOV for the device
 263  * @pdev:  Pointer to pci device.
 264  *
 265  * Function disables SRIOV for the pci device.
 266  *
 267  * Return: 0 on success, error code otherwise.
 268  */
 269 void adf_disable_sriov(struct adf_accel_dev *accel_dev)
 270 {
 271         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 272         struct adf_bar *pmisc =
 273                         &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
 274         void __iomem *pmisc_addr = pmisc->virt_addr;
 275         int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev));
 276         struct adf_accel_vf_info *vf;
 277         u32 reg;
 278         int i;
 279
 280         if (!accel_dev->pf.vf_info)
 281                 return;
 282
 283         adf_pf2vf_notify_restarting(accel_dev);
 284
 285         pci_disable_sriov(accel_to_pci_dev(accel_dev));
 286
 287         /* Disable VF to PF interrupts */
 288         adf_disable_vf2pf_interrupts(accel_dev, 0xFFFFFFFF);
 289
 290         /* Clear Valid bits in ME Thread to PCIe Function Mapping Group A */
 291         for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) {
 292                 reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i);
 293                 reg &= ~ME2FUNCTION_MAP_VALID;
 294                 WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg);
 295         }
 296
 297         /* Clear Valid bits in ME Thread to PCIe Function Mapping Group B */
 298         for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) {
 299                 reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i);
 300                 reg &= ~ME2FUNCTION_MAP_VALID;
 301                 WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg);
 302         }
 303
 304         for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) {
 305                 tasklet_disable(&vf->vf2pf_bh_tasklet);
 306                 tasklet_kill(&vf->vf2pf_bh_tasklet);
 307                 mutex_destroy(&vf->pf2vf_lock);
 308         }
 309
 310         kfree(accel_dev->pf.vf_info);
 311         accel_dev->pf.vf_info = NULL;
 312
 313         if (pf2vf_resp_wq) {
 314                 destroy_workqueue(pf2vf_resp_wq);
 315                 pf2vf_resp_wq = NULL;
 316         }
 317 }
 318 EXPORT_SYMBOL_GPL(adf_disable_sriov);
 319
 320 /**
 321  * adf_sriov_configure() - Enable SRIOV for the device
 322  * @pdev:  Pointer to pci device.
 323  *
 324  * Function enables SRIOV for the pci device.
 325  *
 326  * Return: 0 on success, error code otherwise.
 327  */
 328 int adf_sriov_configure(struct pci_dev *pdev, int numvfs)
 329 {
 330         struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
 331         int totalvfs = pci_sriov_get_totalvfs(pdev);
 332         unsigned long val;
 333         int ret;
 334
 335         if (!accel_dev) {
 336                 dev_err(&pdev->dev, "Failed to find accel_dev\n");
 337                 return -EFAULT;
 338         }
 339
 340         if (!iommu_present(&pci_bus_type)) {
 341                 dev_err(&pdev->dev,
 342                         "IOMMU must be enabled for SR-IOV to work\n");
 343                 return -EINVAL;
 344         }
 345
 346         if (accel_dev->pf.vf_info) {
 347                 dev_info(&pdev->dev, "Already enabled for this device\n");
 348                 return -EINVAL;
 349         }
 350
 351         if (adf_dev_started(accel_dev)) {
 352                 if (adf_devmgr_in_reset(accel_dev) ||
 353                     adf_dev_in_use(accel_dev)) {
 354                         dev_err(&GET_DEV(accel_dev), "Device busy\n");
 355                         return -EBUSY;
 356                 }
 357
 358                 if (adf_dev_stop(accel_dev)) {
 359                         dev_err(&GET_DEV(accel_dev),
 360                                 "Failed to stop qat_dev%d\n",
 361                                 accel_dev->accel_id);
 362                         return -EFAULT;
 363                 }
 364
 365                 adf_dev_shutdown(accel_dev);
 366         }
 367
 368         if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
 369                 return -EFAULT;
 370         val = 0;
 371         if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 372                                         ADF_NUM_CY, (void *)&val, ADF_DEC))
 373                 return -EFAULT;
 374
 375         set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
 376
 377         /* Allocate memory for VF info structs */
 378         accel_dev->pf.vf_info = kcalloc(totalvfs,
 379                                         sizeof(struct adf_accel_vf_info),
 380                                         GFP_KERNEL);
 381         if (!accel_dev->pf.vf_info)
 382                 return -ENOMEM;
 383
 384         if (adf_dev_init(accel_dev)) {
 385                 dev_err(&GET_DEV(accel_dev), "Failed to init qat_dev%d\n",
 386                         accel_dev->accel_id);
 387                 return -EFAULT;
 388         }
 389
 390         if (adf_dev_start(accel_dev)) {
 391                 dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n",
 392                         accel_dev->accel_id);
 393                 return -EFAULT;
 394         }
 395
 396         ret = adf_enable_sriov(accel_dev);
 397         if (ret)
 398                 return ret;
 399
 400         return numvfs;
 401 }
 402 EXPORT_SYMBOL_GPL(adf_sriov_configure);