1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
7 #include <linux/io-64-nonatomic-lo-hi.h>
8 #include <linux/dmaengine.h>
10 #include <linux/msi.h>
11 #include <uapi/linux/idxd.h>
12 #include "../dmaengine.h"
14 #include "registers.h"
16 static void idxd_cmd_exec(struct idxd_device
*idxd
, int cmd_code
, u32 operand
,
18 static void idxd_device_wqs_clear_state(struct idxd_device
*idxd
);
19 static void idxd_wq_disable_cleanup(struct idxd_wq
*wq
);
21 /* Interrupt control bits */
22 void idxd_mask_msix_vector(struct idxd_device
*idxd
, int vec_id
)
24 struct irq_data
*data
= irq_get_irq_data(idxd
->irq_entries
[vec_id
].vector
);
26 pci_msi_mask_irq(data
);
29 void idxd_mask_msix_vectors(struct idxd_device
*idxd
)
31 struct pci_dev
*pdev
= idxd
->pdev
;
32 int msixcnt
= pci_msix_vec_count(pdev
);
35 for (i
= 0; i
< msixcnt
; i
++)
36 idxd_mask_msix_vector(idxd
, i
);
39 void idxd_unmask_msix_vector(struct idxd_device
*idxd
, int vec_id
)
41 struct irq_data
*data
= irq_get_irq_data(idxd
->irq_entries
[vec_id
].vector
);
43 pci_msi_unmask_irq(data
);
46 void idxd_unmask_error_interrupts(struct idxd_device
*idxd
)
48 union genctrl_reg genctrl
;
50 genctrl
.bits
= ioread32(idxd
->reg_base
+ IDXD_GENCTRL_OFFSET
);
51 genctrl
.softerr_int_en
= 1;
52 genctrl
.halt_int_en
= 1;
53 iowrite32(genctrl
.bits
, idxd
->reg_base
+ IDXD_GENCTRL_OFFSET
);
56 void idxd_mask_error_interrupts(struct idxd_device
*idxd
)
58 union genctrl_reg genctrl
;
60 genctrl
.bits
= ioread32(idxd
->reg_base
+ IDXD_GENCTRL_OFFSET
);
61 genctrl
.softerr_int_en
= 0;
62 genctrl
.halt_int_en
= 0;
63 iowrite32(genctrl
.bits
, idxd
->reg_base
+ IDXD_GENCTRL_OFFSET
);
66 static void free_hw_descs(struct idxd_wq
*wq
)
70 for (i
= 0; i
< wq
->num_descs
; i
++)
71 kfree(wq
->hw_descs
[i
]);
76 static int alloc_hw_descs(struct idxd_wq
*wq
, int num
)
78 struct device
*dev
= &wq
->idxd
->pdev
->dev
;
80 int node
= dev_to_node(dev
);
82 wq
->hw_descs
= kcalloc_node(num
, sizeof(struct dsa_hw_desc
*),
87 for (i
= 0; i
< num
; i
++) {
88 wq
->hw_descs
[i
] = kzalloc_node(sizeof(*wq
->hw_descs
[i
]),
90 if (!wq
->hw_descs
[i
]) {
99 static void free_descs(struct idxd_wq
*wq
)
103 for (i
= 0; i
< wq
->num_descs
; i
++)
109 static int alloc_descs(struct idxd_wq
*wq
, int num
)
111 struct device
*dev
= &wq
->idxd
->pdev
->dev
;
113 int node
= dev_to_node(dev
);
115 wq
->descs
= kcalloc_node(num
, sizeof(struct idxd_desc
*),
120 for (i
= 0; i
< num
; i
++) {
121 wq
->descs
[i
] = kzalloc_node(sizeof(*wq
->descs
[i
]),
132 /* WQ control bits */
133 int idxd_wq_alloc_resources(struct idxd_wq
*wq
)
135 struct idxd_device
*idxd
= wq
->idxd
;
136 struct device
*dev
= &idxd
->pdev
->dev
;
137 int rc
, num_descs
, i
;
141 if (wq
->type
!= IDXD_WQT_KERNEL
)
144 wq
->num_descs
= wq
->size
;
145 num_descs
= wq
->size
;
147 rc
= alloc_hw_descs(wq
, num_descs
);
151 align
= idxd
->data
->align
;
152 wq
->compls_size
= num_descs
* idxd
->data
->compl_size
+ align
;
153 wq
->compls_raw
= dma_alloc_coherent(dev
, wq
->compls_size
,
154 &wq
->compls_addr_raw
, GFP_KERNEL
);
155 if (!wq
->compls_raw
) {
157 goto fail_alloc_compls
;
160 /* Adjust alignment */
161 wq
->compls_addr
= (wq
->compls_addr_raw
+ (align
- 1)) & ~(align
- 1);
162 tmp
= (u64
)wq
->compls_raw
;
163 tmp
= (tmp
+ (align
- 1)) & ~(align
- 1);
164 wq
->compls
= (struct dsa_completion_record
*)tmp
;
166 rc
= alloc_descs(wq
, num_descs
);
168 goto fail_alloc_descs
;
170 rc
= sbitmap_queue_init_node(&wq
->sbq
, num_descs
, -1, false, GFP_KERNEL
,
173 goto fail_sbitmap_init
;
175 for (i
= 0; i
< num_descs
; i
++) {
176 struct idxd_desc
*desc
= wq
->descs
[i
];
178 desc
->hw
= wq
->hw_descs
[i
];
179 if (idxd
->data
->type
== IDXD_TYPE_DSA
)
180 desc
->completion
= &wq
->compls
[i
];
181 else if (idxd
->data
->type
== IDXD_TYPE_IAX
)
182 desc
->iax_completion
= &wq
->iax_compls
[i
];
183 desc
->compl_dma
= wq
->compls_addr
+ idxd
->data
->compl_size
* i
;
194 dma_free_coherent(dev
, wq
->compls_size
, wq
->compls_raw
,
195 wq
->compls_addr_raw
);
201 void idxd_wq_free_resources(struct idxd_wq
*wq
)
203 struct device
*dev
= &wq
->idxd
->pdev
->dev
;
205 if (wq
->type
!= IDXD_WQT_KERNEL
)
210 dma_free_coherent(dev
, wq
->compls_size
, wq
->compls_raw
,
211 wq
->compls_addr_raw
);
212 sbitmap_queue_free(&wq
->sbq
);
215 int idxd_wq_enable(struct idxd_wq
*wq
)
217 struct idxd_device
*idxd
= wq
->idxd
;
218 struct device
*dev
= &idxd
->pdev
->dev
;
221 if (wq
->state
== IDXD_WQ_ENABLED
) {
222 dev_dbg(dev
, "WQ %d already enabled\n", wq
->id
);
226 idxd_cmd_exec(idxd
, IDXD_CMD_ENABLE_WQ
, wq
->id
, &status
);
228 if (status
!= IDXD_CMDSTS_SUCCESS
&&
229 status
!= IDXD_CMDSTS_ERR_WQ_ENABLED
) {
230 dev_dbg(dev
, "WQ enable failed: %#x\n", status
);
234 wq
->state
= IDXD_WQ_ENABLED
;
235 dev_dbg(dev
, "WQ %d enabled\n", wq
->id
);
239 int idxd_wq_disable(struct idxd_wq
*wq
, bool reset_config
)
241 struct idxd_device
*idxd
= wq
->idxd
;
242 struct device
*dev
= &idxd
->pdev
->dev
;
245 dev_dbg(dev
, "Disabling WQ %d\n", wq
->id
);
247 if (wq
->state
!= IDXD_WQ_ENABLED
) {
248 dev_dbg(dev
, "WQ %d in wrong state: %d\n", wq
->id
, wq
->state
);
252 operand
= BIT(wq
->id
% 16) | ((wq
->id
/ 16) << 16);
253 idxd_cmd_exec(idxd
, IDXD_CMD_DISABLE_WQ
, operand
, &status
);
255 if (status
!= IDXD_CMDSTS_SUCCESS
) {
256 dev_dbg(dev
, "WQ disable failed: %#x\n", status
);
261 idxd_wq_disable_cleanup(wq
);
262 wq
->state
= IDXD_WQ_DISABLED
;
263 dev_dbg(dev
, "WQ %d disabled\n", wq
->id
);
267 void idxd_wq_drain(struct idxd_wq
*wq
)
269 struct idxd_device
*idxd
= wq
->idxd
;
270 struct device
*dev
= &idxd
->pdev
->dev
;
273 if (wq
->state
!= IDXD_WQ_ENABLED
) {
274 dev_dbg(dev
, "WQ %d in wrong state: %d\n", wq
->id
, wq
->state
);
278 dev_dbg(dev
, "Draining WQ %d\n", wq
->id
);
279 operand
= BIT(wq
->id
% 16) | ((wq
->id
/ 16) << 16);
280 idxd_cmd_exec(idxd
, IDXD_CMD_DRAIN_WQ
, operand
, NULL
);
283 void idxd_wq_reset(struct idxd_wq
*wq
)
285 struct idxd_device
*idxd
= wq
->idxd
;
286 struct device
*dev
= &idxd
->pdev
->dev
;
289 if (wq
->state
!= IDXD_WQ_ENABLED
) {
290 dev_dbg(dev
, "WQ %d in wrong state: %d\n", wq
->id
, wq
->state
);
294 operand
= BIT(wq
->id
% 16) | ((wq
->id
/ 16) << 16);
295 idxd_cmd_exec(idxd
, IDXD_CMD_RESET_WQ
, operand
, NULL
);
296 idxd_wq_disable_cleanup(wq
);
297 wq
->state
= IDXD_WQ_DISABLED
;
300 int idxd_wq_map_portal(struct idxd_wq
*wq
)
302 struct idxd_device
*idxd
= wq
->idxd
;
303 struct pci_dev
*pdev
= idxd
->pdev
;
304 struct device
*dev
= &pdev
->dev
;
305 resource_size_t start
;
307 start
= pci_resource_start(pdev
, IDXD_WQ_BAR
);
308 start
+= idxd_get_wq_portal_full_offset(wq
->id
, IDXD_PORTAL_LIMITED
);
310 wq
->portal
= devm_ioremap(dev
, start
, IDXD_PORTAL_SIZE
);
317 void idxd_wq_unmap_portal(struct idxd_wq
*wq
)
319 struct device
*dev
= &wq
->idxd
->pdev
->dev
;
321 devm_iounmap(dev
, wq
->portal
);
325 void idxd_wqs_unmap_portal(struct idxd_device
*idxd
)
329 for (i
= 0; i
< idxd
->max_wqs
; i
++) {
330 struct idxd_wq
*wq
= idxd
->wqs
[i
];
333 idxd_wq_unmap_portal(wq
);
337 int idxd_wq_set_pasid(struct idxd_wq
*wq
, int pasid
)
339 struct idxd_device
*idxd
= wq
->idxd
;
345 rc
= idxd_wq_disable(wq
, false);
349 offset
= WQCFG_OFFSET(idxd
, wq
->id
, WQCFG_PASID_IDX
);
350 spin_lock_irqsave(&idxd
->dev_lock
, flags
);
351 wqcfg
.bits
[WQCFG_PASID_IDX
] = ioread32(idxd
->reg_base
+ offset
);
354 iowrite32(wqcfg
.bits
[WQCFG_PASID_IDX
], idxd
->reg_base
+ offset
);
355 spin_unlock_irqrestore(&idxd
->dev_lock
, flags
);
357 rc
= idxd_wq_enable(wq
);
364 int idxd_wq_disable_pasid(struct idxd_wq
*wq
)
366 struct idxd_device
*idxd
= wq
->idxd
;
372 rc
= idxd_wq_disable(wq
, false);
376 offset
= WQCFG_OFFSET(idxd
, wq
->id
, WQCFG_PASID_IDX
);
377 spin_lock_irqsave(&idxd
->dev_lock
, flags
);
378 wqcfg
.bits
[WQCFG_PASID_IDX
] = ioread32(idxd
->reg_base
+ offset
);
381 iowrite32(wqcfg
.bits
[WQCFG_PASID_IDX
], idxd
->reg_base
+ offset
);
382 spin_unlock_irqrestore(&idxd
->dev_lock
, flags
);
384 rc
= idxd_wq_enable(wq
);
391 static void idxd_wq_disable_cleanup(struct idxd_wq
*wq
)
393 struct idxd_device
*idxd
= wq
->idxd
;
395 lockdep_assert_held(&wq
->wq_lock
);
396 memset(wq
->wqcfg
, 0, idxd
->wqcfg_size
);
397 wq
->type
= IDXD_WQT_NONE
;
403 clear_bit(WQ_FLAG_DEDICATED
, &wq
->flags
);
404 memset(wq
->name
, 0, WQ_NAME_SIZE
);
407 static void idxd_wq_ref_release(struct percpu_ref
*ref
)
409 struct idxd_wq
*wq
= container_of(ref
, struct idxd_wq
, wq_active
);
411 complete(&wq
->wq_dead
);
414 int idxd_wq_init_percpu_ref(struct idxd_wq
*wq
)
418 memset(&wq
->wq_active
, 0, sizeof(wq
->wq_active
));
419 rc
= percpu_ref_init(&wq
->wq_active
, idxd_wq_ref_release
, 0, GFP_KERNEL
);
422 reinit_completion(&wq
->wq_dead
);
426 void idxd_wq_quiesce(struct idxd_wq
*wq
)
428 percpu_ref_kill(&wq
->wq_active
);
429 wait_for_completion(&wq
->wq_dead
);
430 percpu_ref_exit(&wq
->wq_active
);
433 /* Device control bits */
434 static inline bool idxd_is_enabled(struct idxd_device
*idxd
)
436 union gensts_reg gensts
;
438 gensts
.bits
= ioread32(idxd
->reg_base
+ IDXD_GENSTATS_OFFSET
);
440 if (gensts
.state
== IDXD_DEVICE_STATE_ENABLED
)
445 static inline bool idxd_device_is_halted(struct idxd_device
*idxd
)
447 union gensts_reg gensts
;
449 gensts
.bits
= ioread32(idxd
->reg_base
+ IDXD_GENSTATS_OFFSET
);
451 return (gensts
.state
== IDXD_DEVICE_STATE_HALT
);
455 * This is function is only used for reset during probe and will
456 * poll for completion. Once the device is setup with interrupts,
457 * all commands will be done via interrupt completion.
459 int idxd_device_init_reset(struct idxd_device
*idxd
)
461 struct device
*dev
= &idxd
->pdev
->dev
;
462 union idxd_command_reg cmd
;
465 if (idxd_device_is_halted(idxd
)) {
466 dev_warn(&idxd
->pdev
->dev
, "Device is HALTED!\n");
470 memset(&cmd
, 0, sizeof(cmd
));
471 cmd
.cmd
= IDXD_CMD_RESET_DEVICE
;
472 dev_dbg(dev
, "%s: sending reset for init.\n", __func__
);
473 spin_lock_irqsave(&idxd
->cmd_lock
, flags
);
474 iowrite32(cmd
.bits
, idxd
->reg_base
+ IDXD_CMD_OFFSET
);
476 while (ioread32(idxd
->reg_base
+ IDXD_CMDSTS_OFFSET
) &
479 spin_unlock_irqrestore(&idxd
->cmd_lock
, flags
);
483 static void idxd_cmd_exec(struct idxd_device
*idxd
, int cmd_code
, u32 operand
,
486 union idxd_command_reg cmd
;
487 DECLARE_COMPLETION_ONSTACK(done
);
491 if (idxd_device_is_halted(idxd
)) {
492 dev_warn(&idxd
->pdev
->dev
, "Device is HALTED!\n");
494 *status
= IDXD_CMDSTS_HW_ERR
;
498 memset(&cmd
, 0, sizeof(cmd
));
500 cmd
.operand
= operand
;
503 spin_lock_irqsave(&idxd
->cmd_lock
, flags
);
504 wait_event_lock_irq(idxd
->cmd_waitq
,
505 !test_bit(IDXD_FLAG_CMD_RUNNING
, &idxd
->flags
),
508 dev_dbg(&idxd
->pdev
->dev
, "%s: sending cmd: %#x op: %#x\n",
509 __func__
, cmd_code
, operand
);
511 idxd
->cmd_status
= 0;
512 __set_bit(IDXD_FLAG_CMD_RUNNING
, &idxd
->flags
);
513 idxd
->cmd_done
= &done
;
514 iowrite32(cmd
.bits
, idxd
->reg_base
+ IDXD_CMD_OFFSET
);
517 * After command submitted, release lock and go to sleep until
518 * the command completes via interrupt.
520 spin_unlock_irqrestore(&idxd
->cmd_lock
, flags
);
521 wait_for_completion(&done
);
522 stat
= ioread32(idxd
->reg_base
+ IDXD_CMDSTS_OFFSET
);
523 spin_lock_irqsave(&idxd
->cmd_lock
, flags
);
526 idxd
->cmd_status
= stat
& GENMASK(7, 0);
528 __clear_bit(IDXD_FLAG_CMD_RUNNING
, &idxd
->flags
);
529 /* Wake up other pending commands */
530 wake_up(&idxd
->cmd_waitq
);
531 spin_unlock_irqrestore(&idxd
->cmd_lock
, flags
);
534 int idxd_device_enable(struct idxd_device
*idxd
)
536 struct device
*dev
= &idxd
->pdev
->dev
;
539 if (idxd_is_enabled(idxd
)) {
540 dev_dbg(dev
, "Device already enabled\n");
544 idxd_cmd_exec(idxd
, IDXD_CMD_ENABLE_DEVICE
, 0, &status
);
546 /* If the command is successful or if the device was enabled */
547 if (status
!= IDXD_CMDSTS_SUCCESS
&&
548 status
!= IDXD_CMDSTS_ERR_DEV_ENABLED
) {
549 dev_dbg(dev
, "%s: err_code: %#x\n", __func__
, status
);
553 idxd
->state
= IDXD_DEV_ENABLED
;
557 int idxd_device_disable(struct idxd_device
*idxd
)
559 struct device
*dev
= &idxd
->pdev
->dev
;
563 if (!idxd_is_enabled(idxd
)) {
564 dev_dbg(dev
, "Device is not enabled\n");
568 idxd_cmd_exec(idxd
, IDXD_CMD_DISABLE_DEVICE
, 0, &status
);
570 /* If the command is successful or if the device was disabled */
571 if (status
!= IDXD_CMDSTS_SUCCESS
&&
572 !(status
& IDXD_CMDSTS_ERR_DIS_DEV_EN
)) {
573 dev_dbg(dev
, "%s: err_code: %#x\n", __func__
, status
);
577 spin_lock_irqsave(&idxd
->dev_lock
, flags
);
578 idxd_device_clear_state(idxd
);
579 idxd
->state
= IDXD_DEV_DISABLED
;
580 spin_unlock_irqrestore(&idxd
->dev_lock
, flags
);
584 void idxd_device_reset(struct idxd_device
*idxd
)
588 idxd_cmd_exec(idxd
, IDXD_CMD_RESET_DEVICE
, 0, NULL
);
589 spin_lock_irqsave(&idxd
->dev_lock
, flags
);
590 idxd_device_clear_state(idxd
);
591 idxd
->state
= IDXD_DEV_DISABLED
;
592 spin_unlock_irqrestore(&idxd
->dev_lock
, flags
);
595 void idxd_device_drain_pasid(struct idxd_device
*idxd
, int pasid
)
597 struct device
*dev
= &idxd
->pdev
->dev
;
601 dev_dbg(dev
, "cmd: %u operand: %#x\n", IDXD_CMD_DRAIN_PASID
, operand
);
602 idxd_cmd_exec(idxd
, IDXD_CMD_DRAIN_PASID
, operand
, NULL
);
603 dev_dbg(dev
, "pasid %d drained\n", pasid
);
606 int idxd_device_request_int_handle(struct idxd_device
*idxd
, int idx
, int *handle
,
607 enum idxd_interrupt_type irq_type
)
609 struct device
*dev
= &idxd
->pdev
->dev
;
612 if (!(idxd
->hw
.cmd_cap
& BIT(IDXD_CMD_REQUEST_INT_HANDLE
)))
615 dev_dbg(dev
, "get int handle, idx %d\n", idx
);
617 operand
= idx
& GENMASK(15, 0);
618 if (irq_type
== IDXD_IRQ_IMS
)
619 operand
|= CMD_INT_HANDLE_IMS
;
621 dev_dbg(dev
, "cmd: %u operand: %#x\n", IDXD_CMD_REQUEST_INT_HANDLE
, operand
);
623 idxd_cmd_exec(idxd
, IDXD_CMD_REQUEST_INT_HANDLE
, operand
, &status
);
625 if ((status
& IDXD_CMDSTS_ERR_MASK
) != IDXD_CMDSTS_SUCCESS
) {
626 dev_dbg(dev
, "request int handle failed: %#x\n", status
);
630 *handle
= (status
>> IDXD_CMDSTS_RES_SHIFT
) & GENMASK(15, 0);
632 dev_dbg(dev
, "int handle acquired: %u\n", *handle
);
636 int idxd_device_release_int_handle(struct idxd_device
*idxd
, int handle
,
637 enum idxd_interrupt_type irq_type
)
639 struct device
*dev
= &idxd
->pdev
->dev
;
641 union idxd_command_reg cmd
;
644 if (!(idxd
->hw
.cmd_cap
& BIT(IDXD_CMD_RELEASE_INT_HANDLE
)))
647 dev_dbg(dev
, "release int handle, handle %d\n", handle
);
649 memset(&cmd
, 0, sizeof(cmd
));
650 operand
= handle
& GENMASK(15, 0);
652 if (irq_type
== IDXD_IRQ_IMS
)
653 operand
|= CMD_INT_HANDLE_IMS
;
655 cmd
.cmd
= IDXD_CMD_RELEASE_INT_HANDLE
;
656 cmd
.operand
= operand
;
658 dev_dbg(dev
, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE
, operand
);
660 spin_lock_irqsave(&idxd
->cmd_lock
, flags
);
661 iowrite32(cmd
.bits
, idxd
->reg_base
+ IDXD_CMD_OFFSET
);
663 while (ioread32(idxd
->reg_base
+ IDXD_CMDSTS_OFFSET
) & IDXD_CMDSTS_ACTIVE
)
665 status
= ioread32(idxd
->reg_base
+ IDXD_CMDSTS_OFFSET
);
666 spin_unlock_irqrestore(&idxd
->cmd_lock
, flags
);
668 if ((status
& IDXD_CMDSTS_ERR_MASK
) != IDXD_CMDSTS_SUCCESS
) {
669 dev_dbg(dev
, "release int handle failed: %#x\n", status
);
673 dev_dbg(dev
, "int handle released.\n");
677 /* Device configuration bits */
678 static void idxd_engines_clear_state(struct idxd_device
*idxd
)
680 struct idxd_engine
*engine
;
683 lockdep_assert_held(&idxd
->dev_lock
);
684 for (i
= 0; i
< idxd
->max_engines
; i
++) {
685 engine
= idxd
->engines
[i
];
686 engine
->group
= NULL
;
690 static void idxd_groups_clear_state(struct idxd_device
*idxd
)
692 struct idxd_group
*group
;
695 lockdep_assert_held(&idxd
->dev_lock
);
696 for (i
= 0; i
< idxd
->max_groups
; i
++) {
697 group
= idxd
->groups
[i
];
698 memset(&group
->grpcfg
, 0, sizeof(group
->grpcfg
));
699 group
->num_engines
= 0;
701 group
->use_token_limit
= false;
702 group
->tokens_allowed
= 0;
703 group
->tokens_reserved
= 0;
709 static void idxd_device_wqs_clear_state(struct idxd_device
*idxd
)
713 lockdep_assert_held(&idxd
->dev_lock
);
714 for (i
= 0; i
< idxd
->max_wqs
; i
++) {
715 struct idxd_wq
*wq
= idxd
->wqs
[i
];
717 if (wq
->state
== IDXD_WQ_ENABLED
) {
718 idxd_wq_disable_cleanup(wq
);
719 wq
->state
= IDXD_WQ_DISABLED
;
724 void idxd_device_clear_state(struct idxd_device
*idxd
)
726 idxd_groups_clear_state(idxd
);
727 idxd_engines_clear_state(idxd
);
728 idxd_device_wqs_clear_state(idxd
);
731 void idxd_msix_perm_setup(struct idxd_device
*idxd
)
733 union msix_perm mperm
;
736 msixcnt
= pci_msix_vec_count(idxd
->pdev
);
741 mperm
.pasid
= idxd
->pasid
;
742 mperm
.pasid_en
= device_pasid_enabled(idxd
);
743 for (i
= 1; i
< msixcnt
; i
++)
744 iowrite32(mperm
.bits
, idxd
->reg_base
+ idxd
->msix_perm_offset
+ i
* 8);
747 void idxd_msix_perm_clear(struct idxd_device
*idxd
)
749 union msix_perm mperm
;
752 msixcnt
= pci_msix_vec_count(idxd
->pdev
);
757 for (i
= 1; i
< msixcnt
; i
++)
758 iowrite32(mperm
.bits
, idxd
->reg_base
+ idxd
->msix_perm_offset
+ i
* 8);
761 static void idxd_group_config_write(struct idxd_group
*group
)
763 struct idxd_device
*idxd
= group
->idxd
;
764 struct device
*dev
= &idxd
->pdev
->dev
;
768 dev_dbg(dev
, "Writing group %d cfg registers\n", group
->id
);
771 for (i
= 0; i
< GRPWQCFG_STRIDES
; i
++) {
772 grpcfg_offset
= GRPWQCFG_OFFSET(idxd
, group
->id
, i
);
773 iowrite64(group
->grpcfg
.wqs
[i
], idxd
->reg_base
+ grpcfg_offset
);
774 dev_dbg(dev
, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
775 group
->id
, i
, grpcfg_offset
,
776 ioread64(idxd
->reg_base
+ grpcfg_offset
));
779 /* setup GRPENGCFG */
780 grpcfg_offset
= GRPENGCFG_OFFSET(idxd
, group
->id
);
781 iowrite64(group
->grpcfg
.engines
, idxd
->reg_base
+ grpcfg_offset
);
782 dev_dbg(dev
, "GRPCFG engs[%d: %#x]: %#llx\n", group
->id
,
783 grpcfg_offset
, ioread64(idxd
->reg_base
+ grpcfg_offset
));
786 grpcfg_offset
= GRPFLGCFG_OFFSET(idxd
, group
->id
);
787 iowrite32(group
->grpcfg
.flags
.bits
, idxd
->reg_base
+ grpcfg_offset
);
788 dev_dbg(dev
, "GRPFLAGS flags[%d: %#x]: %#x\n",
789 group
->id
, grpcfg_offset
,
790 ioread32(idxd
->reg_base
+ grpcfg_offset
));
793 static int idxd_groups_config_write(struct idxd_device
*idxd
)
796 union gencfg_reg reg
;
798 struct device
*dev
= &idxd
->pdev
->dev
;
800 /* Setup bandwidth token limit */
801 if (idxd
->token_limit
) {
802 reg
.bits
= ioread32(idxd
->reg_base
+ IDXD_GENCFG_OFFSET
);
803 reg
.token_limit
= idxd
->token_limit
;
804 iowrite32(reg
.bits
, idxd
->reg_base
+ IDXD_GENCFG_OFFSET
);
807 dev_dbg(dev
, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET
,
808 ioread32(idxd
->reg_base
+ IDXD_GENCFG_OFFSET
));
810 for (i
= 0; i
< idxd
->max_groups
; i
++) {
811 struct idxd_group
*group
= idxd
->groups
[i
];
813 idxd_group_config_write(group
);
819 static int idxd_wq_config_write(struct idxd_wq
*wq
)
821 struct idxd_device
*idxd
= wq
->idxd
;
822 struct device
*dev
= &idxd
->pdev
->dev
;
830 * Instead of memset the entire shadow copy of WQCFG, copy from the hardware after
831 * wq reset. This will copy back the sticky values that are present on some devices.
833 for (i
= 0; i
< WQCFG_STRIDES(idxd
); i
++) {
834 wq_offset
= WQCFG_OFFSET(idxd
, wq
->id
, i
);
835 wq
->wqcfg
->bits
[i
] = ioread32(idxd
->reg_base
+ wq_offset
);
839 wq
->wqcfg
->wq_size
= wq
->size
;
842 dev_warn(dev
, "Incorrect work queue size: 0\n");
847 wq
->wqcfg
->wq_thresh
= wq
->threshold
;
850 wq
->wqcfg
->priv
= !!(wq
->type
== IDXD_WQT_KERNEL
);
851 if (wq_dedicated(wq
))
854 if (device_pasid_enabled(idxd
)) {
855 wq
->wqcfg
->pasid_en
= 1;
856 if (wq
->type
== IDXD_WQT_KERNEL
&& wq_dedicated(wq
))
857 wq
->wqcfg
->pasid
= idxd
->pasid
;
860 wq
->wqcfg
->priority
= wq
->priority
;
862 if (idxd
->hw
.gen_cap
.block_on_fault
&&
863 test_bit(WQ_FLAG_BLOCK_ON_FAULT
, &wq
->flags
))
866 if (idxd
->hw
.wq_cap
.wq_ats_support
)
867 wq
->wqcfg
->wq_ats_disable
= wq
->ats_dis
;
870 wq
->wqcfg
->max_xfer_shift
= ilog2(wq
->max_xfer_bytes
);
871 wq
->wqcfg
->max_batch_shift
= ilog2(wq
->max_batch_size
);
873 dev_dbg(dev
, "WQ %d CFGs\n", wq
->id
);
874 for (i
= 0; i
< WQCFG_STRIDES(idxd
); i
++) {
875 wq_offset
= WQCFG_OFFSET(idxd
, wq
->id
, i
);
876 iowrite32(wq
->wqcfg
->bits
[i
], idxd
->reg_base
+ wq_offset
);
877 dev_dbg(dev
, "WQ[%d][%d][%#x]: %#x\n",
878 wq
->id
, i
, wq_offset
,
879 ioread32(idxd
->reg_base
+ wq_offset
));
885 static int idxd_wqs_config_write(struct idxd_device
*idxd
)
889 for (i
= 0; i
< idxd
->max_wqs
; i
++) {
890 struct idxd_wq
*wq
= idxd
->wqs
[i
];
892 rc
= idxd_wq_config_write(wq
);
900 static void idxd_group_flags_setup(struct idxd_device
*idxd
)
904 /* TC-A 0 and TC-B 1 should be defaults */
905 for (i
= 0; i
< idxd
->max_groups
; i
++) {
906 struct idxd_group
*group
= idxd
->groups
[i
];
908 if (group
->tc_a
== -1)
909 group
->tc_a
= group
->grpcfg
.flags
.tc_a
= 0;
911 group
->grpcfg
.flags
.tc_a
= group
->tc_a
;
912 if (group
->tc_b
== -1)
913 group
->tc_b
= group
->grpcfg
.flags
.tc_b
= 1;
915 group
->grpcfg
.flags
.tc_b
= group
->tc_b
;
916 group
->grpcfg
.flags
.use_token_limit
= group
->use_token_limit
;
917 group
->grpcfg
.flags
.tokens_reserved
= group
->tokens_reserved
;
918 if (group
->tokens_allowed
)
919 group
->grpcfg
.flags
.tokens_allowed
=
920 group
->tokens_allowed
;
922 group
->grpcfg
.flags
.tokens_allowed
= idxd
->max_tokens
;
926 static int idxd_engines_setup(struct idxd_device
*idxd
)
929 struct idxd_engine
*eng
;
930 struct idxd_group
*group
;
932 for (i
= 0; i
< idxd
->max_groups
; i
++) {
933 group
= idxd
->groups
[i
];
934 group
->grpcfg
.engines
= 0;
937 for (i
= 0; i
< idxd
->max_engines
; i
++) {
938 eng
= idxd
->engines
[i
];
944 group
->grpcfg
.engines
|= BIT(eng
->id
);
954 static int idxd_wqs_setup(struct idxd_device
*idxd
)
957 struct idxd_group
*group
;
958 int i
, j
, configured
= 0;
959 struct device
*dev
= &idxd
->pdev
->dev
;
961 for (i
= 0; i
< idxd
->max_groups
; i
++) {
962 group
= idxd
->groups
[i
];
963 for (j
= 0; j
< 4; j
++)
964 group
->grpcfg
.wqs
[j
] = 0;
967 for (i
= 0; i
< idxd
->max_wqs
; i
++) {
976 if (wq_shared(wq
) && !device_swq_supported(idxd
)) {
977 dev_warn(dev
, "No shared wq support but configured.\n");
981 group
->grpcfg
.wqs
[wq
->id
/ 64] |= BIT(wq
->id
% 64);
991 int idxd_device_config(struct idxd_device
*idxd
)
995 lockdep_assert_held(&idxd
->dev_lock
);
996 rc
= idxd_wqs_setup(idxd
);
1000 rc
= idxd_engines_setup(idxd
);
1004 idxd_group_flags_setup(idxd
);
1006 rc
= idxd_wqs_config_write(idxd
);
1010 rc
= idxd_groups_config_write(idxd
);
1017 static int idxd_wq_load_config(struct idxd_wq
*wq
)
1019 struct idxd_device
*idxd
= wq
->idxd
;
1020 struct device
*dev
= &idxd
->pdev
->dev
;
1024 wqcfg_offset
= WQCFG_OFFSET(idxd
, wq
->id
, 0);
1025 memcpy_fromio(wq
->wqcfg
, idxd
->reg_base
+ wqcfg_offset
, idxd
->wqcfg_size
);
1027 wq
->size
= wq
->wqcfg
->wq_size
;
1028 wq
->threshold
= wq
->wqcfg
->wq_thresh
;
1029 if (wq
->wqcfg
->priv
)
1030 wq
->type
= IDXD_WQT_KERNEL
;
1032 /* The driver does not support shared WQ mode in read-only config yet */
1033 if (wq
->wqcfg
->mode
== 0 || wq
->wqcfg
->pasid_en
)
1036 set_bit(WQ_FLAG_DEDICATED
, &wq
->flags
);
1038 wq
->priority
= wq
->wqcfg
->priority
;
1040 for (i
= 0; i
< WQCFG_STRIDES(idxd
); i
++) {
1041 wqcfg_offset
= WQCFG_OFFSET(idxd
, wq
->id
, i
);
1042 dev_dbg(dev
, "WQ[%d][%d][%#x]: %#x\n", wq
->id
, i
, wqcfg_offset
, wq
->wqcfg
->bits
[i
]);
1048 static void idxd_group_load_config(struct idxd_group
*group
)
1050 struct idxd_device
*idxd
= group
->idxd
;
1051 struct device
*dev
= &idxd
->pdev
->dev
;
1052 int i
, j
, grpcfg_offset
;
1055 * Load WQS bit fields
1056 * Iterate through all 256 bits 64 bits at a time
1058 for (i
= 0; i
< GRPWQCFG_STRIDES
; i
++) {
1061 grpcfg_offset
= GRPWQCFG_OFFSET(idxd
, group
->id
, i
);
1062 group
->grpcfg
.wqs
[i
] = ioread64(idxd
->reg_base
+ grpcfg_offset
);
1063 dev_dbg(dev
, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
1064 group
->id
, i
, grpcfg_offset
, group
->grpcfg
.wqs
[i
]);
1066 if (i
* 64 >= idxd
->max_wqs
)
1069 /* Iterate through all 64 bits and check for wq set */
1070 for (j
= 0; j
< 64; j
++) {
1071 int id
= i
* 64 + j
;
1073 /* No need to check beyond max wqs */
1074 if (id
>= idxd
->max_wqs
)
1077 /* Set group assignment for wq if wq bit is set */
1078 if (group
->grpcfg
.wqs
[i
] & BIT(j
)) {
1085 grpcfg_offset
= GRPENGCFG_OFFSET(idxd
, group
->id
);
1086 group
->grpcfg
.engines
= ioread64(idxd
->reg_base
+ grpcfg_offset
);
1087 dev_dbg(dev
, "GRPCFG engs[%d: %#x]: %#llx\n", group
->id
,
1088 grpcfg_offset
, group
->grpcfg
.engines
);
1090 /* Iterate through all 64 bits to check engines set */
1091 for (i
= 0; i
< 64; i
++) {
1092 if (i
>= idxd
->max_engines
)
1095 if (group
->grpcfg
.engines
& BIT(i
)) {
1096 struct idxd_engine
*engine
= idxd
->engines
[i
];
1098 engine
->group
= group
;
1102 grpcfg_offset
= GRPFLGCFG_OFFSET(idxd
, group
->id
);
1103 group
->grpcfg
.flags
.bits
= ioread32(idxd
->reg_base
+ grpcfg_offset
);
1104 dev_dbg(dev
, "GRPFLAGS flags[%d: %#x]: %#x\n",
1105 group
->id
, grpcfg_offset
, group
->grpcfg
.flags
.bits
);
1108 int idxd_device_load_config(struct idxd_device
*idxd
)
1110 union gencfg_reg reg
;
1113 reg
.bits
= ioread32(idxd
->reg_base
+ IDXD_GENCFG_OFFSET
);
1114 idxd
->token_limit
= reg
.token_limit
;
1116 for (i
= 0; i
< idxd
->max_groups
; i
++) {
1117 struct idxd_group
*group
= idxd
->groups
[i
];
1119 idxd_group_load_config(group
);
1122 for (i
= 0; i
< idxd
->max_wqs
; i
++) {
1123 struct idxd_wq
*wq
= idxd
->wqs
[i
];
1125 rc
= idxd_wq_load_config(wq
);
1133 static int __drv_enable_wq(struct idxd_wq
*wq
)
1135 struct idxd_device
*idxd
= wq
->idxd
;
1136 struct device
*dev
= &idxd
->pdev
->dev
;
1137 unsigned long flags
;
1140 lockdep_assert_held(&wq
->wq_lock
);
1142 if (idxd
->state
!= IDXD_DEV_ENABLED
)
1145 if (wq
->state
!= IDXD_WQ_DISABLED
) {
1146 dev_dbg(dev
, "wq %d already enabled.\n", wq
->id
);
1152 dev_dbg(dev
, "wq %d not attached to group.\n", wq
->id
);
1156 if (strlen(wq
->name
) == 0) {
1157 dev_dbg(dev
, "wq %d name not set.\n", wq
->id
);
1161 /* Shared WQ checks */
1162 if (wq_shared(wq
)) {
1163 if (!device_swq_supported(idxd
)) {
1164 dev_dbg(dev
, "PASID not enabled and shared wq.\n");
1168 * Shared wq with the threshold set to 0 means the user
1169 * did not set the threshold or transitioned from a
1170 * dedicated wq but did not set threshold. A value
1171 * of 0 would effectively disable the shared wq. The
1172 * driver does not allow a value of 0 to be set for
1173 * threshold via sysfs.
1175 if (wq
->threshold
== 0) {
1176 dev_dbg(dev
, "Shared wq and threshold 0.\n");
1181 rc
= idxd_wq_alloc_resources(wq
);
1183 dev_dbg(dev
, "wq resource alloc failed\n");
1187 spin_lock_irqsave(&idxd
->dev_lock
, flags
);
1188 if (test_bit(IDXD_FLAG_CONFIGURABLE
, &idxd
->flags
))
1189 rc
= idxd_device_config(idxd
);
1190 spin_unlock_irqrestore(&idxd
->dev_lock
, flags
);
1192 dev_dbg(dev
, "Writing wq %d config failed: %d\n", wq
->id
, rc
);
1196 rc
= idxd_wq_enable(wq
);
1198 dev_dbg(dev
, "wq %d enabling failed: %d\n", wq
->id
, rc
);
1202 rc
= idxd_wq_map_portal(wq
);
1204 dev_dbg(dev
, "wq %d portal mapping failed: %d\n", wq
->id
, rc
);
1205 goto err_map_portal
;
1208 wq
->client_count
= 0;
1210 if (wq
->type
== IDXD_WQT_KERNEL
) {
1211 rc
= idxd_wq_init_percpu_ref(wq
);
1213 dev_dbg(dev
, "wq %d percpu_ref setup failed\n", wq
->id
);
1218 if (is_idxd_wq_dmaengine(wq
)) {
1219 rc
= idxd_register_dma_channel(wq
);
1221 dev_dbg(dev
, "wq %d DMA channel register failed\n", wq
->id
);
1224 } else if (is_idxd_wq_cdev(wq
)) {
1225 rc
= idxd_wq_add_cdev(wq
);
1227 dev_dbg(dev
, "wq %d cdev creation failed\n", wq
->id
);
1232 dev_info(dev
, "wq %s enabled\n", dev_name(wq_confdev(wq
)));
1236 idxd_wq_quiesce(wq
);
1238 idxd_wq_unmap_portal(wq
);
1240 rc
= idxd_wq_disable(wq
, false);
1242 dev_dbg(dev
, "wq %s disable failed\n", dev_name(wq_confdev(wq
)));
1247 int drv_enable_wq(struct idxd_wq
*wq
)
1251 mutex_lock(&wq
->wq_lock
);
1252 rc
= __drv_enable_wq(wq
);
1253 mutex_unlock(&wq
->wq_lock
);
1257 static void __drv_disable_wq(struct idxd_wq
*wq
)
1259 struct idxd_device
*idxd
= wq
->idxd
;
1260 struct device
*dev
= &idxd
->pdev
->dev
;
1262 lockdep_assert_held(&wq
->wq_lock
);
1264 if (wq
->type
== IDXD_WQT_KERNEL
)
1265 idxd_wq_quiesce(wq
);
1267 if (is_idxd_wq_dmaengine(wq
))
1268 idxd_unregister_dma_channel(wq
);
1269 else if (is_idxd_wq_cdev(wq
))
1270 idxd_wq_del_cdev(wq
);
1272 if (idxd_wq_refcount(wq
))
1273 dev_warn(dev
, "Clients has claim on wq %d: %d\n",
1274 wq
->id
, idxd_wq_refcount(wq
));
1276 idxd_wq_unmap_portal(wq
);
1281 idxd_wq_free_resources(wq
);
1282 wq
->client_count
= 0;
1284 dev_info(dev
, "wq %s disabled\n", dev_name(wq_confdev(wq
)));
1287 void drv_disable_wq(struct idxd_wq
*wq
)
1289 mutex_lock(&wq
->wq_lock
);
1290 __drv_disable_wq(wq
);
1291 mutex_unlock(&wq
->wq_lock
);