1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/kernel.h>
4 #include <linux/module.h>
6 #include <linux/init.h>
7 #include <linux/types.h>
9 #include <linux/slab.h>
10 #include <linux/errno.h>
11 #include <linux/device.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/platform_device.h>
14 #include <linux/workqueue.h>
16 #include <adf_accel_devices.h>
17 #include <adf_common_drv.h>
19 #include "adf_c3xxx_hw_data.h"
21 static const struct pci_device_id adf_pci_tbl
[] = {
22 { PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_QAT_C3XXX
), },
25 MODULE_DEVICE_TABLE(pci
, adf_pci_tbl
);
27 static int adf_probe(struct pci_dev
*dev
, const struct pci_device_id
*ent
);
28 static void adf_remove(struct pci_dev
*dev
);
30 static struct pci_driver adf_driver
= {
31 .id_table
= adf_pci_tbl
,
32 .name
= ADF_C3XXX_DEVICE_NAME
,
35 .sriov_configure
= adf_sriov_configure
,
38 static void adf_cleanup_pci_dev(struct adf_accel_dev
*accel_dev
)
40 pci_release_regions(accel_dev
->accel_pci_dev
.pci_dev
);
41 pci_disable_device(accel_dev
->accel_pci_dev
.pci_dev
);
44 static void adf_cleanup_accel(struct adf_accel_dev
*accel_dev
)
46 struct adf_accel_pci
*accel_pci_dev
= &accel_dev
->accel_pci_dev
;
49 for (i
= 0; i
< ADF_PCI_MAX_BARS
; i
++) {
50 struct adf_bar
*bar
= &accel_pci_dev
->pci_bars
[i
];
53 pci_iounmap(accel_pci_dev
->pci_dev
, bar
->virt_addr
);
56 if (accel_dev
->hw_device
) {
57 switch (accel_pci_dev
->pci_dev
->device
) {
58 case PCI_DEVICE_ID_INTEL_QAT_C3XXX
:
59 adf_clean_hw_data_c3xxx(accel_dev
->hw_device
);
64 kfree(accel_dev
->hw_device
);
65 accel_dev
->hw_device
= NULL
;
67 adf_cfg_dev_remove(accel_dev
);
68 debugfs_remove(accel_dev
->debugfs_dir
);
69 adf_devmgr_rm_dev(accel_dev
, NULL
);
72 static int adf_probe(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
74 struct adf_accel_dev
*accel_dev
;
75 struct adf_accel_pci
*accel_pci_dev
;
76 struct adf_hw_device_data
*hw_data
;
77 char name
[ADF_DEVICE_NAME_LENGTH
];
78 unsigned int i
, bar_nr
;
79 unsigned long bar_mask
;
82 switch (ent
->device
) {
83 case PCI_DEVICE_ID_INTEL_QAT_C3XXX
:
86 dev_err(&pdev
->dev
, "Invalid device 0x%x.\n", ent
->device
);
90 if (num_possible_nodes() > 1 && dev_to_node(&pdev
->dev
) < 0) {
91 /* If the accelerator is connected to a node with no memory
92 * there is no point in using the accelerator since the remote
93 * memory transaction will be very slow. */
94 dev_err(&pdev
->dev
, "Invalid NUMA configuration.\n");
98 accel_dev
= kzalloc_node(sizeof(*accel_dev
), GFP_KERNEL
,
99 dev_to_node(&pdev
->dev
));
103 INIT_LIST_HEAD(&accel_dev
->crypto_list
);
104 accel_pci_dev
= &accel_dev
->accel_pci_dev
;
105 accel_pci_dev
->pci_dev
= pdev
;
107 /* Add accel device to accel table.
108 * This should be called before adf_cleanup_accel is called */
109 if (adf_devmgr_add_dev(accel_dev
, NULL
)) {
110 dev_err(&pdev
->dev
, "Failed to add new accelerator device.\n");
115 accel_dev
->owner
= THIS_MODULE
;
116 /* Allocate and configure device configuration structure */
117 hw_data
= kzalloc_node(sizeof(*hw_data
), GFP_KERNEL
,
118 dev_to_node(&pdev
->dev
));
124 accel_dev
->hw_device
= hw_data
;
125 adf_init_hw_data_c3xxx(accel_dev
->hw_device
);
126 pci_read_config_byte(pdev
, PCI_REVISION_ID
, &accel_pci_dev
->revid
);
127 pci_read_config_dword(pdev
, ADF_DEVICE_FUSECTL_OFFSET
,
129 pci_read_config_dword(pdev
, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET
,
132 /* Get Accelerators and Accelerators Engines masks */
133 hw_data
->accel_mask
= hw_data
->get_accel_mask(hw_data
);
134 hw_data
->ae_mask
= hw_data
->get_ae_mask(hw_data
);
135 accel_pci_dev
->sku
= hw_data
->get_sku(hw_data
);
136 /* If the device has no acceleration engines then ignore it. */
137 if (!hw_data
->accel_mask
|| !hw_data
->ae_mask
||
138 ((~hw_data
->ae_mask
) & 0x01)) {
139 dev_err(&pdev
->dev
, "No acceleration units found");
144 /* Create dev top level debugfs entry */
145 snprintf(name
, sizeof(name
), "%s%s_%s", ADF_DEVICE_NAME_PREFIX
,
146 hw_data
->dev_class
->name
, pci_name(pdev
));
148 accel_dev
->debugfs_dir
= debugfs_create_dir(name
, NULL
);
150 /* Create device configuration table */
151 ret
= adf_cfg_dev_add(accel_dev
);
155 /* enable PCI device */
156 if (pci_enable_device(pdev
)) {
161 /* set dma identifier */
162 ret
= dma_set_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(64));
164 dev_err(&pdev
->dev
, "No usable DMA configuration\n");
165 goto out_err_disable
;
168 if (pci_request_regions(pdev
, ADF_C3XXX_DEVICE_NAME
)) {
170 goto out_err_disable
;
173 /* Get accelerator capabilities mask */
174 hw_data
->accel_capabilities_mask
= hw_data
->get_accel_cap(accel_dev
);
176 /* Find and map all the device's BARS */
178 bar_mask
= pci_select_bars(pdev
, IORESOURCE_MEM
);
179 for_each_set_bit(bar_nr
, &bar_mask
, ADF_PCI_MAX_BARS
* 2) {
180 struct adf_bar
*bar
= &accel_pci_dev
->pci_bars
[i
++];
182 bar
->base_addr
= pci_resource_start(pdev
, bar_nr
);
185 bar
->size
= pci_resource_len(pdev
, bar_nr
);
186 bar
->virt_addr
= pci_iomap(accel_pci_dev
->pci_dev
, bar_nr
, 0);
187 if (!bar
->virt_addr
) {
188 dev_err(&pdev
->dev
, "Failed to map BAR %d\n", bar_nr
);
190 goto out_err_free_reg
;
193 pci_set_master(pdev
);
195 if (adf_enable_aer(accel_dev
)) {
196 dev_err(&pdev
->dev
, "Failed to enable aer\n");
198 goto out_err_free_reg
;
201 if (pci_save_state(pdev
)) {
202 dev_err(&pdev
->dev
, "Failed to save pci state\n");
204 goto out_err_free_reg
;
207 ret
= qat_crypto_dev_config(accel_dev
);
209 goto out_err_free_reg
;
211 ret
= adf_dev_init(accel_dev
);
213 goto out_err_dev_shutdown
;
215 ret
= adf_dev_start(accel_dev
);
217 goto out_err_dev_stop
;
222 adf_dev_stop(accel_dev
);
223 out_err_dev_shutdown
:
224 adf_dev_shutdown(accel_dev
);
226 pci_release_regions(accel_pci_dev
->pci_dev
);
228 pci_disable_device(accel_pci_dev
->pci_dev
);
230 adf_cleanup_accel(accel_dev
);
235 static void adf_remove(struct pci_dev
*pdev
)
237 struct adf_accel_dev
*accel_dev
= adf_devmgr_pci_to_accel_dev(pdev
);
240 pr_err("QAT: Driver removal failed\n");
243 adf_dev_stop(accel_dev
);
244 adf_dev_shutdown(accel_dev
);
245 adf_disable_aer(accel_dev
);
246 adf_cleanup_accel(accel_dev
);
247 adf_cleanup_pci_dev(accel_dev
);
251 static int __init
adfdrv_init(void)
253 request_module("intel_qat");
255 if (pci_register_driver(&adf_driver
)) {
256 pr_err("QAT: Driver initialization failed\n");
262 static void __exit
adfdrv_release(void)
264 pci_unregister_driver(&adf_driver
);
267 module_init(adfdrv_init
);
268 module_exit(adfdrv_release
);
270 MODULE_LICENSE("Dual BSD/GPL");
271 MODULE_AUTHOR("Intel");
272 MODULE_FIRMWARE(ADF_C3XXX_FW
);
273 MODULE_FIRMWARE(ADF_C3XXX_MMP
);
274 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
275 MODULE_VERSION(ADF_DRV_VERSION
);