]>
Commit | Line | Data |
---|---|---|
bfe1d560 DJ |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ | |
3 | #include <linux/init.h> | |
4 | #include <linux/kernel.h> | |
5 | #include <linux/module.h> | |
6 | #include <linux/slab.h> | |
7 | #include <linux/pci.h> | |
8 | #include <linux/interrupt.h> | |
9 | #include <linux/delay.h> | |
10 | #include <linux/dma-mapping.h> | |
11 | #include <linux/workqueue.h> | |
12 | #include <linux/aer.h> | |
13 | #include <linux/fs.h> | |
14 | #include <linux/io-64-nonatomic-lo-hi.h> | |
15 | #include <linux/device.h> | |
16 | #include <linux/idr.h> | |
8e50d392 DJ |
17 | #include <linux/intel-svm.h> |
18 | #include <linux/iommu.h> | |
bfe1d560 | 19 | #include <uapi/linux/idxd.h> |
8f47d1a5 DJ |
20 | #include <linux/dmaengine.h> |
21 | #include "../dmaengine.h" | |
bfe1d560 DJ |
22 | #include "registers.h" |
23 | #include "idxd.h" | |
0bde4444 | 24 | #include "perfmon.h" |
bfe1d560 DJ |
25 | |
26 | MODULE_VERSION(IDXD_DRIVER_VERSION); | |
27 | MODULE_LICENSE("GPL v2"); | |
28 | MODULE_AUTHOR("Intel Corporation"); | |
d9e5481f | 29 | MODULE_IMPORT_NS(IDXD); |
bfe1d560 | 30 | |
03d939c7 DJ |
31 | static bool sva = true; |
32 | module_param(sva, bool, 0644); | |
33 | MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); | |
34 | ||
ade8a86b DJ |
35 | bool tc_override; |
36 | module_param(tc_override, bool, 0644); | |
37 | MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); | |
38 | ||
bfe1d560 DJ |
39 | #define DRV_NAME "idxd" |
40 | ||
8e50d392 | 41 | bool support_enqcmd; |
4b73e4eb | 42 | DEFINE_IDA(idxd_ida); |
bfe1d560 | 43 | |
435b512d DJ |
44 | static struct idxd_driver_data idxd_driver_data[] = { |
45 | [IDXD_TYPE_DSA] = { | |
46 | .name_prefix = "dsa", | |
47 | .type = IDXD_TYPE_DSA, | |
48 | .compl_size = sizeof(struct dsa_completion_record), | |
49 | .align = 32, | |
50 | .dev_type = &dsa_device_type, | |
51 | }, | |
52 | [IDXD_TYPE_IAX] = { | |
53 | .name_prefix = "iax", | |
54 | .type = IDXD_TYPE_IAX, | |
55 | .compl_size = sizeof(struct iax_completion_record), | |
56 | .align = 64, | |
57 | .dev_type = &iax_device_type, | |
58 | }, | |
59 | }; | |
60 | ||
bfe1d560 DJ |
61 | static struct pci_device_id idxd_pci_tbl[] = { |
62 | /* DSA ver 1.0 platforms */ | |
435b512d | 63 | { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, |
f25b4638 DJ |
64 | |
65 | /* IAX ver 1.0 platforms */ | |
435b512d | 66 | { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, |
bfe1d560 DJ |
67 | { 0, } |
68 | }; | |
69 | MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); | |
70 | ||
bfe1d560 DJ |
71 | static int idxd_setup_interrupts(struct idxd_device *idxd) |
72 | { | |
73 | struct pci_dev *pdev = idxd->pdev; | |
74 | struct device *dev = &pdev->dev; | |
bfe1d560 DJ |
75 | struct idxd_irq_entry *irq_entry; |
76 | int i, msixcnt; | |
77 | int rc = 0; | |
78 | ||
79 | msixcnt = pci_msix_vec_count(pdev); | |
80 | if (msixcnt < 0) { | |
81 | dev_err(dev, "Not MSI-X interrupt capable.\n"); | |
5fc8e85f | 82 | return -ENOSPC; |
bfe1d560 DJ |
83 | } |
84 | ||
5fc8e85f DJ |
85 | rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); |
86 | if (rc != msixcnt) { | |
87 | dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); | |
88 | return -ENOSPC; | |
bfe1d560 DJ |
89 | } |
90 | dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); | |
91 | ||
92 | /* | |
93 | * We implement 1 completion list per MSI-X entry except for | |
94 | * entry 0, which is for errors and others. | |
95 | */ | |
47c16ac2 DJ |
96 | idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry), |
97 | GFP_KERNEL, dev_to_node(dev)); | |
bfe1d560 DJ |
98 | if (!idxd->irq_entries) { |
99 | rc = -ENOMEM; | |
5fc8e85f | 100 | goto err_irq_entries; |
bfe1d560 DJ |
101 | } |
102 | ||
103 | for (i = 0; i < msixcnt; i++) { | |
104 | idxd->irq_entries[i].id = i; | |
105 | idxd->irq_entries[i].idxd = idxd; | |
5fc8e85f | 106 | idxd->irq_entries[i].vector = pci_irq_vector(pdev, i); |
e4f4d8cd | 107 | spin_lock_init(&idxd->irq_entries[i].list_lock); |
bfe1d560 DJ |
108 | } |
109 | ||
d5c10e0f DJ |
110 | idxd_msix_perm_setup(idxd); |
111 | ||
bfe1d560 | 112 | irq_entry = &idxd->irq_entries[0]; |
a1610461 | 113 | rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread, |
5fc8e85f | 114 | 0, "idxd-misc", irq_entry); |
bfe1d560 DJ |
115 | if (rc < 0) { |
116 | dev_err(dev, "Failed to allocate misc interrupt.\n"); | |
5fc8e85f | 117 | goto err_misc_irq; |
bfe1d560 DJ |
118 | } |
119 | ||
5fc8e85f | 120 | dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector); |
bfe1d560 DJ |
121 | |
122 | /* first MSI-X entry is not for wq interrupts */ | |
123 | idxd->num_wq_irqs = msixcnt - 1; | |
124 | ||
125 | for (i = 1; i < msixcnt; i++) { | |
bfe1d560 DJ |
126 | irq_entry = &idxd->irq_entries[i]; |
127 | ||
128 | init_llist_head(&idxd->irq_entries[i].pending_llist); | |
129 | INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); | |
a1610461 | 130 | rc = request_threaded_irq(irq_entry->vector, NULL, |
5fc8e85f | 131 | idxd_wq_thread, 0, "idxd-portal", irq_entry); |
bfe1d560 | 132 | if (rc < 0) { |
5fc8e85f DJ |
133 | dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector); |
134 | goto err_wq_irqs; | |
bfe1d560 | 135 | } |
eb15e715 | 136 | |
5fc8e85f | 137 | dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector); |
eb15e715 DJ |
138 | if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { |
139 | /* | |
140 | * The MSIX vector enumeration starts at 1 with vector 0 being the | |
141 | * misc interrupt that handles non I/O completion events. The | |
142 | * interrupt handles are for IMS enumeration on guest. The misc | |
143 | * interrupt vector does not require a handle and therefore we start | |
144 | * the int_handles at index 0. Since 'i' starts at 1, the first | |
145 | * int_handles index will be 0. | |
146 | */ | |
147 | rc = idxd_device_request_int_handle(idxd, i, &idxd->int_handles[i - 1], | |
148 | IDXD_IRQ_MSIX); | |
149 | if (rc < 0) { | |
150 | free_irq(irq_entry->vector, irq_entry); | |
151 | goto err_wq_irqs; | |
152 | } | |
153 | dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i - 1]); | |
154 | } | |
bfe1d560 DJ |
155 | } |
156 | ||
157 | idxd_unmask_error_interrupts(idxd); | |
bfe1d560 DJ |
158 | return 0; |
159 | ||
5fc8e85f DJ |
160 | err_wq_irqs: |
161 | while (--i >= 0) { | |
162 | irq_entry = &idxd->irq_entries[i]; | |
163 | free_irq(irq_entry->vector, irq_entry); | |
eb15e715 DJ |
164 | if (i != 0) |
165 | idxd_device_release_int_handle(idxd, | |
166 | idxd->int_handles[i], IDXD_IRQ_MSIX); | |
5fc8e85f DJ |
167 | } |
168 | err_misc_irq: | |
bfe1d560 DJ |
169 | /* Disable error interrupt generation */ |
170 | idxd_mask_error_interrupts(idxd); | |
d5c10e0f | 171 | idxd_msix_perm_clear(idxd); |
5fc8e85f DJ |
172 | err_irq_entries: |
173 | pci_free_irq_vectors(pdev); | |
bfe1d560 DJ |
174 | dev_err(dev, "No usable interrupts\n"); |
175 | return rc; | |
176 | } | |
177 | ||
ddf742d4 DJ |
178 | static void idxd_cleanup_interrupts(struct idxd_device *idxd) |
179 | { | |
180 | struct pci_dev *pdev = idxd->pdev; | |
181 | struct idxd_irq_entry *irq_entry; | |
182 | int i, msixcnt; | |
183 | ||
184 | msixcnt = pci_msix_vec_count(pdev); | |
185 | if (msixcnt <= 0) | |
186 | return; | |
187 | ||
188 | irq_entry = &idxd->irq_entries[0]; | |
189 | free_irq(irq_entry->vector, irq_entry); | |
190 | ||
191 | for (i = 1; i < msixcnt; i++) { | |
192 | ||
193 | irq_entry = &idxd->irq_entries[i]; | |
194 | if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) | |
195 | idxd_device_release_int_handle(idxd, idxd->int_handles[i], | |
196 | IDXD_IRQ_MSIX); | |
197 | free_irq(irq_entry->vector, irq_entry); | |
198 | } | |
199 | ||
200 | idxd_mask_error_interrupts(idxd); | |
201 | pci_free_irq_vectors(pdev); | |
202 | } | |
203 | ||
7c5dd23e DJ |
204 | static int idxd_setup_wqs(struct idxd_device *idxd) |
205 | { | |
206 | struct device *dev = &idxd->pdev->dev; | |
207 | struct idxd_wq *wq; | |
700af3a0 | 208 | struct device *conf_dev; |
7c5dd23e DJ |
209 | int i, rc; |
210 | ||
211 | idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), | |
212 | GFP_KERNEL, dev_to_node(dev)); | |
213 | if (!idxd->wqs) | |
214 | return -ENOMEM; | |
215 | ||
216 | for (i = 0; i < idxd->max_wqs; i++) { | |
217 | wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); | |
218 | if (!wq) { | |
219 | rc = -ENOMEM; | |
220 | goto err; | |
221 | } | |
222 | ||
700af3a0 DJ |
223 | idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); |
224 | conf_dev = wq_confdev(wq); | |
7c5dd23e DJ |
225 | wq->id = i; |
226 | wq->idxd = idxd; | |
700af3a0 DJ |
227 | device_initialize(wq_confdev(wq)); |
228 | conf_dev->parent = idxd_confdev(idxd); | |
229 | conf_dev->bus = &dsa_bus_type; | |
230 | conf_dev->type = &idxd_wq_device_type; | |
231 | rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); | |
7c5dd23e | 232 | if (rc < 0) { |
700af3a0 | 233 | put_device(conf_dev); |
7c5dd23e DJ |
234 | goto err; |
235 | } | |
236 | ||
237 | mutex_init(&wq->wq_lock); | |
04922b74 | 238 | init_waitqueue_head(&wq->err_queue); |
93a40a6d | 239 | init_completion(&wq->wq_dead); |
7c5dd23e DJ |
240 | wq->max_xfer_bytes = idxd->max_xfer_bytes; |
241 | wq->max_batch_size = idxd->max_batch_size; | |
242 | wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); | |
243 | if (!wq->wqcfg) { | |
700af3a0 | 244 | put_device(conf_dev); |
7c5dd23e DJ |
245 | rc = -ENOMEM; |
246 | goto err; | |
247 | } | |
248 | idxd->wqs[i] = wq; | |
249 | } | |
250 | ||
251 | return 0; | |
252 | ||
253 | err: | |
700af3a0 DJ |
254 | while (--i >= 0) { |
255 | wq = idxd->wqs[i]; | |
256 | conf_dev = wq_confdev(wq); | |
257 | put_device(conf_dev); | |
258 | } | |
7c5dd23e DJ |
259 | return rc; |
260 | } | |
261 | ||
75b91130 DJ |
262 | static int idxd_setup_engines(struct idxd_device *idxd) |
263 | { | |
264 | struct idxd_engine *engine; | |
265 | struct device *dev = &idxd->pdev->dev; | |
700af3a0 | 266 | struct device *conf_dev; |
75b91130 DJ |
267 | int i, rc; |
268 | ||
269 | idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), | |
270 | GFP_KERNEL, dev_to_node(dev)); | |
271 | if (!idxd->engines) | |
272 | return -ENOMEM; | |
273 | ||
274 | for (i = 0; i < idxd->max_engines; i++) { | |
275 | engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); | |
276 | if (!engine) { | |
277 | rc = -ENOMEM; | |
278 | goto err; | |
279 | } | |
280 | ||
700af3a0 DJ |
281 | idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); |
282 | conf_dev = engine_confdev(engine); | |
75b91130 DJ |
283 | engine->id = i; |
284 | engine->idxd = idxd; | |
700af3a0 DJ |
285 | device_initialize(conf_dev); |
286 | conf_dev->parent = idxd_confdev(idxd); | |
287 | conf_dev->bus = &dsa_bus_type; | |
288 | conf_dev->type = &idxd_engine_device_type; | |
289 | rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); | |
75b91130 | 290 | if (rc < 0) { |
700af3a0 | 291 | put_device(conf_dev); |
75b91130 DJ |
292 | goto err; |
293 | } | |
294 | ||
295 | idxd->engines[i] = engine; | |
296 | } | |
297 | ||
298 | return 0; | |
299 | ||
300 | err: | |
700af3a0 DJ |
301 | while (--i >= 0) { |
302 | engine = idxd->engines[i]; | |
303 | conf_dev = engine_confdev(engine); | |
304 | put_device(conf_dev); | |
305 | } | |
75b91130 DJ |
306 | return rc; |
307 | } | |
308 | ||
defe49f9 | 309 | static int idxd_setup_groups(struct idxd_device *idxd) |
bfe1d560 DJ |
310 | { |
311 | struct device *dev = &idxd->pdev->dev; | |
700af3a0 | 312 | struct device *conf_dev; |
defe49f9 | 313 | struct idxd_group *group; |
7c5dd23e | 314 | int i, rc; |
bfe1d560 | 315 | |
defe49f9 DJ |
316 | idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), |
317 | GFP_KERNEL, dev_to_node(dev)); | |
318 | if (!idxd->groups) | |
319 | return -ENOMEM; | |
320 | ||
321 | for (i = 0; i < idxd->max_groups; i++) { | |
322 | group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); | |
323 | if (!group) { | |
324 | rc = -ENOMEM; | |
325 | goto err; | |
326 | } | |
327 | ||
700af3a0 DJ |
328 | idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); |
329 | conf_dev = group_confdev(group); | |
defe49f9 DJ |
330 | group->id = i; |
331 | group->idxd = idxd; | |
700af3a0 DJ |
332 | device_initialize(conf_dev); |
333 | conf_dev->parent = idxd_confdev(idxd); | |
334 | conf_dev->bus = &dsa_bus_type; | |
335 | conf_dev->type = &idxd_group_device_type; | |
336 | rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); | |
defe49f9 | 337 | if (rc < 0) { |
700af3a0 | 338 | put_device(conf_dev); |
defe49f9 DJ |
339 | goto err; |
340 | } | |
341 | ||
342 | idxd->groups[i] = group; | |
ade8a86b DJ |
343 | if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) { |
344 | group->tc_a = 1; | |
345 | group->tc_b = 1; | |
346 | } else { | |
347 | group->tc_a = -1; | |
348 | group->tc_b = -1; | |
349 | } | |
defe49f9 DJ |
350 | } |
351 | ||
352 | return 0; | |
353 | ||
354 | err: | |
700af3a0 DJ |
355 | while (--i >= 0) { |
356 | group = idxd->groups[i]; | |
357 | put_device(group_confdev(group)); | |
358 | } | |
defe49f9 DJ |
359 | return rc; |
360 | } | |
361 | ||
ddf742d4 DJ |
362 | static void idxd_cleanup_internals(struct idxd_device *idxd) |
363 | { | |
364 | int i; | |
365 | ||
366 | for (i = 0; i < idxd->max_groups; i++) | |
700af3a0 | 367 | put_device(group_confdev(idxd->groups[i])); |
ddf742d4 | 368 | for (i = 0; i < idxd->max_engines; i++) |
700af3a0 | 369 | put_device(engine_confdev(idxd->engines[i])); |
ddf742d4 | 370 | for (i = 0; i < idxd->max_wqs; i++) |
700af3a0 | 371 | put_device(wq_confdev(idxd->wqs[i])); |
ddf742d4 DJ |
372 | destroy_workqueue(idxd->wq); |
373 | } | |
374 | ||
defe49f9 DJ |
375 | static int idxd_setup_internals(struct idxd_device *idxd) |
376 | { | |
377 | struct device *dev = &idxd->pdev->dev; | |
378 | int rc, i; | |
379 | ||
0d5c10b4 | 380 | init_waitqueue_head(&idxd->cmd_waitq); |
7c5dd23e | 381 | |
eb15e715 | 382 | if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { |
33f9f3c3 DJ |
383 | idxd->int_handles = kcalloc_node(idxd->max_wqs, sizeof(int), GFP_KERNEL, |
384 | dev_to_node(dev)); | |
eb15e715 DJ |
385 | if (!idxd->int_handles) |
386 | return -ENOMEM; | |
387 | } | |
388 | ||
7c5dd23e DJ |
389 | rc = idxd_setup_wqs(idxd); |
390 | if (rc < 0) | |
eb15e715 | 391 | goto err_wqs; |
7c5dd23e | 392 | |
75b91130 DJ |
393 | rc = idxd_setup_engines(idxd); |
394 | if (rc < 0) | |
395 | goto err_engine; | |
396 | ||
defe49f9 DJ |
397 | rc = idxd_setup_groups(idxd); |
398 | if (rc < 0) | |
399 | goto err_group; | |
bfe1d560 | 400 | |
0d5c10b4 | 401 | idxd->wq = create_workqueue(dev_name(dev)); |
7c5dd23e DJ |
402 | if (!idxd->wq) { |
403 | rc = -ENOMEM; | |
defe49f9 | 404 | goto err_wkq_create; |
7c5dd23e | 405 | } |
0d5c10b4 | 406 | |
bfe1d560 | 407 | return 0; |
7c5dd23e | 408 | |
defe49f9 DJ |
409 | err_wkq_create: |
410 | for (i = 0; i < idxd->max_groups; i++) | |
700af3a0 | 411 | put_device(group_confdev(idxd->groups[i])); |
defe49f9 | 412 | err_group: |
75b91130 | 413 | for (i = 0; i < idxd->max_engines; i++) |
700af3a0 | 414 | put_device(engine_confdev(idxd->engines[i])); |
75b91130 | 415 | err_engine: |
7c5dd23e | 416 | for (i = 0; i < idxd->max_wqs; i++) |
700af3a0 | 417 | put_device(wq_confdev(idxd->wqs[i])); |
eb15e715 DJ |
418 | err_wqs: |
419 | kfree(idxd->int_handles); | |
7c5dd23e | 420 | return rc; |
bfe1d560 DJ |
421 | } |
422 | ||
423 | static void idxd_read_table_offsets(struct idxd_device *idxd) | |
424 | { | |
425 | union offsets_reg offsets; | |
426 | struct device *dev = &idxd->pdev->dev; | |
427 | ||
428 | offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); | |
2f8417a9 DJ |
429 | offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); |
430 | idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; | |
bfe1d560 | 431 | dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); |
2f8417a9 DJ |
432 | idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; |
433 | dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); | |
434 | idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; | |
435 | dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); | |
436 | idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; | |
bfe1d560 DJ |
437 | dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); |
438 | } | |
439 | ||
440 | static void idxd_read_caps(struct idxd_device *idxd) | |
441 | { | |
442 | struct device *dev = &idxd->pdev->dev; | |
443 | int i; | |
444 | ||
445 | /* reading generic capabilities */ | |
446 | idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); | |
447 | dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); | |
eb15e715 DJ |
448 | |
449 | if (idxd->hw.gen_cap.cmd_cap) { | |
450 | idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); | |
451 | dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); | |
452 | } | |
453 | ||
bfe1d560 DJ |
454 | idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; |
455 | dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); | |
456 | idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; | |
457 | dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); | |
458 | if (idxd->hw.gen_cap.config_en) | |
459 | set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); | |
460 | ||
461 | /* reading group capabilities */ | |
462 | idxd->hw.group_cap.bits = | |
463 | ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); | |
464 | dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); | |
465 | idxd->max_groups = idxd->hw.group_cap.num_groups; | |
466 | dev_dbg(dev, "max groups: %u\n", idxd->max_groups); | |
4f1c25f9 DJ |
467 | idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; |
468 | dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); | |
469 | idxd->nr_rdbufs = idxd->max_rdbufs; | |
bfe1d560 DJ |
470 | |
471 | /* read engine capabilities */ | |
472 | idxd->hw.engine_cap.bits = | |
473 | ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); | |
474 | dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); | |
475 | idxd->max_engines = idxd->hw.engine_cap.num_engines; | |
476 | dev_dbg(dev, "max engines: %u\n", idxd->max_engines); | |
477 | ||
478 | /* read workqueue capabilities */ | |
479 | idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); | |
480 | dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); | |
481 | idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; | |
482 | dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); | |
483 | idxd->max_wqs = idxd->hw.wq_cap.num_wqs; | |
484 | dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); | |
d98793b5 DJ |
485 | idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); |
486 | dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); | |
bfe1d560 DJ |
487 | |
488 | /* reading operation capabilities */ | |
489 | for (i = 0; i < 4; i++) { | |
490 | idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + | |
491 | IDXD_OPCAP_OFFSET + i * sizeof(u64)); | |
492 | dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); | |
493 | } | |
494 | } | |
495 | ||
435b512d | 496 | static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) |
bfe1d560 DJ |
497 | { |
498 | struct device *dev = &pdev->dev; | |
700af3a0 | 499 | struct device *conf_dev; |
bfe1d560 | 500 | struct idxd_device *idxd; |
47c16ac2 | 501 | int rc; |
bfe1d560 | 502 | |
47c16ac2 | 503 | idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); |
bfe1d560 DJ |
504 | if (!idxd) |
505 | return NULL; | |
506 | ||
700af3a0 | 507 | conf_dev = idxd_confdev(idxd); |
bfe1d560 | 508 | idxd->pdev = pdev; |
435b512d | 509 | idxd->data = data; |
700af3a0 | 510 | idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); |
4b73e4eb | 511 | idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); |
47c16ac2 DJ |
512 | if (idxd->id < 0) |
513 | return NULL; | |
514 | ||
700af3a0 DJ |
515 | device_initialize(conf_dev); |
516 | conf_dev->parent = dev; | |
517 | conf_dev->bus = &dsa_bus_type; | |
518 | conf_dev->type = idxd->data->dev_type; | |
519 | rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); | |
47c16ac2 | 520 | if (rc < 0) { |
700af3a0 | 521 | put_device(conf_dev); |
47c16ac2 DJ |
522 | return NULL; |
523 | } | |
524 | ||
bfe1d560 | 525 | spin_lock_init(&idxd->dev_lock); |
53b2ee7f | 526 | spin_lock_init(&idxd->cmd_lock); |
bfe1d560 DJ |
527 | |
528 | return idxd; | |
529 | } | |
530 | ||
8e50d392 DJ |
531 | static int idxd_enable_system_pasid(struct idxd_device *idxd) |
532 | { | |
533 | int flags; | |
534 | unsigned int pasid; | |
535 | struct iommu_sva *sva; | |
536 | ||
537 | flags = SVM_FLAG_SUPERVISOR_MODE; | |
538 | ||
539 | sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags); | |
540 | if (IS_ERR(sva)) { | |
541 | dev_warn(&idxd->pdev->dev, | |
542 | "iommu sva bind failed: %ld\n", PTR_ERR(sva)); | |
543 | return PTR_ERR(sva); | |
544 | } | |
545 | ||
546 | pasid = iommu_sva_get_pasid(sva); | |
547 | if (pasid == IOMMU_PASID_INVALID) { | |
548 | iommu_sva_unbind_device(sva); | |
549 | return -ENODEV; | |
550 | } | |
551 | ||
552 | idxd->sva = sva; | |
553 | idxd->pasid = pasid; | |
554 | dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid); | |
555 | return 0; | |
556 | } | |
557 | ||
558 | static void idxd_disable_system_pasid(struct idxd_device *idxd) | |
559 | { | |
560 | ||
561 | iommu_sva_unbind_device(idxd->sva); | |
562 | idxd->sva = NULL; | |
563 | } | |
564 | ||
bfe1d560 DJ |
565 | static int idxd_probe(struct idxd_device *idxd) |
566 | { | |
567 | struct pci_dev *pdev = idxd->pdev; | |
568 | struct device *dev = &pdev->dev; | |
569 | int rc; | |
570 | ||
571 | dev_dbg(dev, "%s entered and resetting device\n", __func__); | |
89e3becd DJ |
572 | rc = idxd_device_init_reset(idxd); |
573 | if (rc < 0) | |
574 | return rc; | |
575 | ||
bfe1d560 DJ |
576 | dev_dbg(dev, "IDXD reset complete\n"); |
577 | ||
03d939c7 | 578 | if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { |
cf5f86a7 DJ |
579 | rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA); |
580 | if (rc == 0) { | |
581 | rc = idxd_enable_system_pasid(idxd); | |
582 | if (rc < 0) { | |
583 | iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); | |
584 | dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); | |
585 | } else { | |
586 | set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); | |
587 | } | |
588 | } else { | |
589 | dev_warn(dev, "Unable to turn on SVA feature.\n"); | |
590 | } | |
03d939c7 DJ |
591 | } else if (!sva) { |
592 | dev_warn(dev, "User forced SVA off via module param.\n"); | |
8e50d392 DJ |
593 | } |
594 | ||
bfe1d560 DJ |
595 | idxd_read_caps(idxd); |
596 | idxd_read_table_offsets(idxd); | |
597 | ||
598 | rc = idxd_setup_internals(idxd); | |
599 | if (rc) | |
7c5dd23e | 600 | goto err; |
bfe1d560 | 601 | |
8c66bbdc DJ |
602 | /* If the configs are readonly, then load them from device */ |
603 | if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { | |
604 | dev_dbg(dev, "Loading RO device config\n"); | |
605 | rc = idxd_device_load_config(idxd); | |
606 | if (rc < 0) | |
ddf742d4 | 607 | goto err_config; |
8c66bbdc DJ |
608 | } |
609 | ||
bfe1d560 DJ |
610 | rc = idxd_setup_interrupts(idxd); |
611 | if (rc) | |
ddf742d4 | 612 | goto err_config; |
bfe1d560 DJ |
613 | |
614 | dev_dbg(dev, "IDXD interrupt setup complete.\n"); | |
615 | ||
42d279f9 DJ |
616 | idxd->major = idxd_cdev_get_major(idxd); |
617 | ||
0bde4444 TZ |
618 | rc = perfmon_pmu_init(idxd); |
619 | if (rc < 0) | |
620 | dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); | |
621 | ||
bfe1d560 DJ |
622 | dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); |
623 | return 0; | |
624 | ||
ddf742d4 DJ |
625 | err_config: |
626 | idxd_cleanup_internals(idxd); | |
7c5dd23e | 627 | err: |
8e50d392 DJ |
628 | if (device_pasid_enabled(idxd)) |
629 | idxd_disable_system_pasid(idxd); | |
cf5f86a7 | 630 | iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); |
bfe1d560 DJ |
631 | return rc; |
632 | } | |
633 | ||
ddf742d4 DJ |
634 | static void idxd_cleanup(struct idxd_device *idxd) |
635 | { | |
636 | struct device *dev = &idxd->pdev->dev; | |
637 | ||
638 | perfmon_pmu_remove(idxd); | |
639 | idxd_cleanup_interrupts(idxd); | |
640 | idxd_cleanup_internals(idxd); | |
641 | if (device_pasid_enabled(idxd)) | |
642 | idxd_disable_system_pasid(idxd); | |
643 | iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); | |
644 | } | |
645 | ||
bfe1d560 DJ |
646 | static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
647 | { | |
bfe1d560 DJ |
648 | struct device *dev = &pdev->dev; |
649 | struct idxd_device *idxd; | |
435b512d | 650 | struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; |
bfe1d560 | 651 | int rc; |
bfe1d560 | 652 | |
a39c7cd0 | 653 | rc = pci_enable_device(pdev); |
bfe1d560 DJ |
654 | if (rc) |
655 | return rc; | |
656 | ||
8e50d392 | 657 | dev_dbg(dev, "Alloc IDXD context\n"); |
435b512d | 658 | idxd = idxd_alloc(pdev, data); |
a39c7cd0 DJ |
659 | if (!idxd) { |
660 | rc = -ENOMEM; | |
661 | goto err_idxd_alloc; | |
662 | } | |
bfe1d560 | 663 | |
8e50d392 | 664 | dev_dbg(dev, "Mapping BARs\n"); |
a39c7cd0 DJ |
665 | idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); |
666 | if (!idxd->reg_base) { | |
667 | rc = -ENOMEM; | |
668 | goto err_iomap; | |
669 | } | |
bfe1d560 DJ |
670 | |
671 | dev_dbg(dev, "Set DMA masks\n"); | |
53b50458 | 672 | rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); |
bfe1d560 | 673 | if (rc) |
53b50458 | 674 | rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); |
bfe1d560 | 675 | if (rc) |
a39c7cd0 | 676 | goto err; |
bfe1d560 | 677 | |
bfe1d560 DJ |
678 | dev_dbg(dev, "Set PCI master\n"); |
679 | pci_set_master(pdev); | |
680 | pci_set_drvdata(pdev, idxd); | |
681 | ||
682 | idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); | |
683 | rc = idxd_probe(idxd); | |
684 | if (rc) { | |
685 | dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); | |
a39c7cd0 | 686 | goto err; |
bfe1d560 DJ |
687 | } |
688 | ||
47c16ac2 | 689 | rc = idxd_register_devices(idxd); |
c52ca478 DJ |
690 | if (rc) { |
691 | dev_err(dev, "IDXD sysfs setup failed\n"); | |
ddf742d4 | 692 | goto err_dev_register; |
c52ca478 DJ |
693 | } |
694 | ||
bfe1d560 DJ |
695 | dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", |
696 | idxd->hw.version); | |
697 | ||
698 | return 0; | |
a39c7cd0 | 699 | |
ddf742d4 DJ |
700 | err_dev_register: |
701 | idxd_cleanup(idxd); | |
a39c7cd0 DJ |
702 | err: |
703 | pci_iounmap(pdev, idxd->reg_base); | |
704 | err_iomap: | |
700af3a0 | 705 | put_device(idxd_confdev(idxd)); |
a39c7cd0 DJ |
706 | err_idxd_alloc: |
707 | pci_disable_device(pdev); | |
708 | return rc; | |
bfe1d560 DJ |
709 | } |
710 | ||
8f47d1a5 DJ |
711 | static void idxd_flush_pending_llist(struct idxd_irq_entry *ie) |
712 | { | |
713 | struct idxd_desc *desc, *itr; | |
714 | struct llist_node *head; | |
715 | ||
716 | head = llist_del_all(&ie->pending_llist); | |
717 | if (!head) | |
718 | return; | |
719 | ||
720 | llist_for_each_entry_safe(desc, itr, head, llnode) { | |
721 | idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); | |
722 | idxd_free_desc(desc->wq, desc); | |
723 | } | |
724 | } | |
725 | ||
726 | static void idxd_flush_work_list(struct idxd_irq_entry *ie) | |
727 | { | |
728 | struct idxd_desc *desc, *iter; | |
729 | ||
730 | list_for_each_entry_safe(desc, iter, &ie->work_list, list) { | |
731 | list_del(&desc->list); | |
732 | idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); | |
733 | idxd_free_desc(desc->wq, desc); | |
734 | } | |
735 | } | |
736 | ||
5b0c68c4 DJ |
737 | void idxd_wqs_quiesce(struct idxd_device *idxd) |
738 | { | |
739 | struct idxd_wq *wq; | |
740 | int i; | |
741 | ||
742 | for (i = 0; i < idxd->max_wqs; i++) { | |
743 | wq = idxd->wqs[i]; | |
744 | if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) | |
745 | idxd_wq_quiesce(wq); | |
746 | } | |
747 | } | |
748 | ||
eb15e715 DJ |
749 | static void idxd_release_int_handles(struct idxd_device *idxd) |
750 | { | |
751 | struct device *dev = &idxd->pdev->dev; | |
752 | int i, rc; | |
753 | ||
754 | for (i = 0; i < idxd->num_wq_irqs; i++) { | |
755 | if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) { | |
756 | rc = idxd_device_release_int_handle(idxd, idxd->int_handles[i], | |
757 | IDXD_IRQ_MSIX); | |
758 | if (rc < 0) | |
759 | dev_warn(dev, "irq handle %d release failed\n", | |
760 | idxd->int_handles[i]); | |
761 | else | |
762 | dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i]); | |
763 | } | |
764 | } | |
765 | } | |
766 | ||
bfe1d560 DJ |
767 | static void idxd_shutdown(struct pci_dev *pdev) |
768 | { | |
769 | struct idxd_device *idxd = pci_get_drvdata(pdev); | |
770 | int rc, i; | |
771 | struct idxd_irq_entry *irq_entry; | |
772 | int msixcnt = pci_msix_vec_count(pdev); | |
bfe1d560 | 773 | |
bfe1d560 | 774 | rc = idxd_device_disable(idxd); |
bfe1d560 DJ |
775 | if (rc) |
776 | dev_err(&pdev->dev, "Disabling device failed\n"); | |
777 | ||
778 | dev_dbg(&pdev->dev, "%s called\n", __func__); | |
779 | idxd_mask_msix_vectors(idxd); | |
780 | idxd_mask_error_interrupts(idxd); | |
781 | ||
782 | for (i = 0; i < msixcnt; i++) { | |
783 | irq_entry = &idxd->irq_entries[i]; | |
5fc8e85f | 784 | synchronize_irq(irq_entry->vector); |
bfe1d560 DJ |
785 | if (i == 0) |
786 | continue; | |
8f47d1a5 DJ |
787 | idxd_flush_pending_llist(irq_entry); |
788 | idxd_flush_work_list(irq_entry); | |
bfe1d560 | 789 | } |
49c4959f | 790 | flush_workqueue(idxd->wq); |
bfe1d560 DJ |
791 | } |
792 | ||
793 | static void idxd_remove(struct pci_dev *pdev) | |
794 | { | |
795 | struct idxd_device *idxd = pci_get_drvdata(pdev); | |
49c4959f DJ |
796 | struct idxd_irq_entry *irq_entry; |
797 | int msixcnt = pci_msix_vec_count(pdev); | |
798 | int i; | |
bfe1d560 | 799 | |
26ba12df DJ |
800 | idxd_unregister_devices(idxd); |
801 | /* | |
802 | * When ->release() is called for the idxd->conf_dev, it frees all the memory related | |
803 | * to the idxd context. The driver still needs those bits in order to do the rest of | |
804 | * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref | |
805 | * on the device here to hold off the freeing while allowing the idxd sub-driver | |
806 | * to unbind. | |
807 | */ | |
808 | get_device(idxd_confdev(idxd)); | |
809 | device_unregister(idxd_confdev(idxd)); | |
bfe1d560 | 810 | idxd_shutdown(pdev); |
8e50d392 DJ |
811 | if (device_pasid_enabled(idxd)) |
812 | idxd_disable_system_pasid(idxd); | |
49c4959f DJ |
813 | |
814 | for (i = 0; i < msixcnt; i++) { | |
815 | irq_entry = &idxd->irq_entries[i]; | |
816 | free_irq(irq_entry->vector, irq_entry); | |
817 | } | |
818 | idxd_msix_perm_clear(idxd); | |
819 | idxd_release_int_handles(idxd); | |
820 | pci_free_irq_vectors(pdev); | |
821 | pci_iounmap(pdev, idxd->reg_base); | |
cf5f86a7 | 822 | iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); |
49c4959f DJ |
823 | pci_disable_device(pdev); |
824 | destroy_workqueue(idxd->wq); | |
825 | perfmon_pmu_remove(idxd); | |
26ba12df | 826 | put_device(idxd_confdev(idxd)); |
bfe1d560 DJ |
827 | } |
828 | ||
829 | static struct pci_driver idxd_pci_driver = { | |
830 | .name = DRV_NAME, | |
831 | .id_table = idxd_pci_tbl, | |
832 | .probe = idxd_pci_probe, | |
833 | .remove = idxd_remove, | |
834 | .shutdown = idxd_shutdown, | |
835 | }; | |
836 | ||
837 | static int __init idxd_init_module(void) | |
838 | { | |
4b73e4eb | 839 | int err; |
bfe1d560 DJ |
840 | |
841 | /* | |
8e50d392 | 842 | * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in |
bfe1d560 DJ |
843 | * enumerating the device. We can not utilize it. |
844 | */ | |
74b2fc88 | 845 | if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { |
bfe1d560 DJ |
846 | pr_warn("idxd driver failed to load without MOVDIR64B.\n"); |
847 | return -ENODEV; | |
848 | } | |
849 | ||
74b2fc88 | 850 | if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) |
8e50d392 DJ |
851 | pr_warn("Platform does not have ENQCMD(S) support.\n"); |
852 | else | |
853 | support_enqcmd = true; | |
bfe1d560 | 854 | |
0bde4444 TZ |
855 | perfmon_init(); |
856 | ||
034b3290 DJ |
857 | err = idxd_driver_register(&idxd_drv); |
858 | if (err < 0) | |
859 | goto err_idxd_driver_register; | |
860 | ||
0cda4f69 DJ |
861 | err = idxd_driver_register(&idxd_dmaengine_drv); |
862 | if (err < 0) | |
863 | goto err_idxd_dmaengine_driver_register; | |
864 | ||
448c3de8 DJ |
865 | err = idxd_driver_register(&idxd_user_drv); |
866 | if (err < 0) | |
867 | goto err_idxd_user_driver_register; | |
868 | ||
42d279f9 DJ |
869 | err = idxd_cdev_register(); |
870 | if (err) | |
871 | goto err_cdev_register; | |
872 | ||
bfe1d560 DJ |
873 | err = pci_register_driver(&idxd_pci_driver); |
874 | if (err) | |
c52ca478 | 875 | goto err_pci_register; |
bfe1d560 DJ |
876 | |
877 | return 0; | |
c52ca478 DJ |
878 | |
879 | err_pci_register: | |
42d279f9 DJ |
880 | idxd_cdev_remove(); |
881 | err_cdev_register: | |
448c3de8 DJ |
882 | idxd_driver_unregister(&idxd_user_drv); |
883 | err_idxd_user_driver_register: | |
0cda4f69 DJ |
884 | idxd_driver_unregister(&idxd_dmaengine_drv); |
885 | err_idxd_dmaengine_driver_register: | |
034b3290 DJ |
886 | idxd_driver_unregister(&idxd_drv); |
887 | err_idxd_driver_register: | |
c52ca478 | 888 | return err; |
bfe1d560 DJ |
889 | } |
890 | module_init(idxd_init_module); | |
891 | ||
892 | static void __exit idxd_exit_module(void) | |
893 | { | |
448c3de8 | 894 | idxd_driver_unregister(&idxd_user_drv); |
0cda4f69 | 895 | idxd_driver_unregister(&idxd_dmaengine_drv); |
034b3290 | 896 | idxd_driver_unregister(&idxd_drv); |
bfe1d560 | 897 | pci_unregister_driver(&idxd_pci_driver); |
42d279f9 | 898 | idxd_cdev_remove(); |
0bde4444 | 899 | perfmon_exit(); |
bfe1d560 DJ |
900 | } |
901 | module_exit(idxd_exit_module); |