1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
6 #include <linux/sbitmap.h>
7 #include <linux/dmaengine.h>
8 #include <linux/percpu-rwsem.h>
9 #include <linux/wait.h>
10 #include <linux/cdev.h>
11 #include <linux/idr.h>
12 #include "registers.h"
14 #define IDXD_DRIVER_VERSION "1.00"
16 extern struct kmem_cache
*idxd_desc_pool
;
21 #define IDXD_REG_TIMEOUT 50
22 #define IDXD_DRAIN_TIMEOUT 5000
25 IDXD_TYPE_UNKNOWN
= -1,
31 #define IDXD_NAME_SIZE 128
33 struct idxd_device_driver
{
34 struct device_driver drv
;
37 struct idxd_irq_entry
{
38 struct idxd_device
*idxd
;
41 struct llist_head pending_llist
;
42 struct list_head work_list
;
44 * Lock to protect access between irq thread process descriptor
45 * and irq thread processing error descriptor.
51 struct device conf_dev
;
52 struct idxd_device
*idxd
;
64 #define IDXD_MAX_PRIORITY 0xf
72 WQ_FLAG_DEDICATED
= 0,
73 WQ_FLAG_BLOCK_ON_FAULT
,
89 #define IDXD_ALLOCATED_BATCH_SIZE 128U
90 #define WQ_NAME_SIZE 1024
91 #define WQ_TYPE_SIZE 10
98 enum idxd_complete_type
{
99 IDXD_COMPLETE_NORMAL
= 0,
101 IDXD_COMPLETE_DEV_FAIL
,
104 struct idxd_dma_chan
{
105 struct dma_chan chan
;
110 void __iomem
*portal
;
111 struct percpu_ref wq_active
;
112 struct completion wq_dead
;
113 struct device conf_dev
;
114 struct idxd_cdev
*idxd_cdev
;
115 struct wait_queue_head err_queue
;
116 struct idxd_device
*idxd
;
118 enum idxd_wq_type type
;
119 struct idxd_group
*group
;
121 struct mutex wq_lock
; /* mutex for workqueue */
125 enum idxd_wq_state state
;
128 u32 vec_ptr
; /* interrupt steering */
129 struct dsa_hw_desc
**hw_descs
;
132 struct dsa_completion_record
*compls
;
133 struct iax_completion_record
*iax_compls
;
136 dma_addr_t compls_addr
;
137 dma_addr_t compls_addr_raw
;
139 struct idxd_desc
**descs
;
140 struct sbitmap_queue sbq
;
141 struct idxd_dma_chan
*idxd_chan
;
142 char name
[WQ_NAME_SIZE
+ 1];
149 struct device conf_dev
;
151 struct idxd_group
*group
;
152 struct idxd_device
*idxd
;
155 /* shadow registers */
158 union gen_cap_reg gen_cap
;
159 union wq_cap_reg wq_cap
;
160 union group_cap_reg group_cap
;
161 union engine_cap_reg engine_cap
;
166 enum idxd_device_state
{
167 IDXD_DEV_HALTED
= -1,
168 IDXD_DEV_DISABLED
= 0,
173 enum idxd_device_flag
{
174 IDXD_FLAG_CONFIGURABLE
= 0,
175 IDXD_FLAG_CMD_RUNNING
,
176 IDXD_FLAG_PASID_ENABLED
,
179 struct idxd_dma_dev
{
180 struct idxd_device
*idxd
;
181 struct dma_device dma
;
184 struct idxd_driver_data
{
185 const char *name_prefix
;
187 struct device_type
*dev_type
;
193 struct device conf_dev
;
194 struct idxd_driver_data
*data
;
195 struct list_head list
;
197 enum idxd_device_state state
;
203 struct pci_dev
*pdev
;
204 void __iomem
*reg_base
;
206 spinlock_t dev_lock
; /* spinlock for device */
207 spinlock_t cmd_lock
; /* spinlock for device commands */
208 struct completion
*cmd_done
;
209 struct idxd_group
**groups
;
210 struct idxd_wq
**wqs
;
211 struct idxd_engine
**engines
;
213 struct iommu_sva
*sva
;
218 u32 msix_perm_offset
;
231 int nr_tokens
; /* non-reserved tokens */
232 unsigned int wqcfg_size
;
234 union sw_err_reg sw_err
;
235 wait_queue_head_t cmd_waitq
;
237 struct idxd_irq_entry
*irq_entries
;
239 struct idxd_dma_dev
*idxd_dma
;
240 struct workqueue_struct
*wq
;
241 struct work_struct work
;
246 /* IDXD software descriptor */
249 struct dsa_hw_desc
*hw
;
250 struct iax_hw_desc
*iax_hw
;
254 struct dsa_completion_record
*completion
;
255 struct iax_completion_record
*iax_completion
;
257 dma_addr_t compl_dma
;
258 struct dma_async_tx_descriptor txd
;
259 struct llist_node llnode
;
260 struct list_head list
;
267 #define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
268 #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
270 extern struct bus_type dsa_bus_type
;
271 extern struct bus_type iax_bus_type
;
273 extern bool support_enqcmd
;
274 extern struct ida idxd_ida
;
275 extern struct device_type dsa_device_type
;
276 extern struct device_type iax_device_type
;
277 extern struct device_type idxd_wq_device_type
;
278 extern struct device_type idxd_engine_device_type
;
279 extern struct device_type idxd_group_device_type
;
281 static inline bool is_dsa_dev(struct device
*dev
)
283 return dev
->type
== &dsa_device_type
;
286 static inline bool is_iax_dev(struct device
*dev
)
288 return dev
->type
== &iax_device_type
;
291 static inline bool is_idxd_dev(struct device
*dev
)
293 return is_dsa_dev(dev
) || is_iax_dev(dev
);
296 static inline bool is_idxd_wq_dev(struct device
*dev
)
298 return dev
->type
== &idxd_wq_device_type
;
301 static inline bool is_idxd_wq_dmaengine(struct idxd_wq
*wq
)
303 if (wq
->type
== IDXD_WQT_KERNEL
&& strcmp(wq
->name
, "dmaengine") == 0)
308 static inline bool is_idxd_wq_cdev(struct idxd_wq
*wq
)
310 return wq
->type
== IDXD_WQT_USER
;
313 static inline bool wq_dedicated(struct idxd_wq
*wq
)
315 return test_bit(WQ_FLAG_DEDICATED
, &wq
->flags
);
318 static inline bool wq_shared(struct idxd_wq
*wq
)
320 return !test_bit(WQ_FLAG_DEDICATED
, &wq
->flags
);
323 static inline bool device_pasid_enabled(struct idxd_device
*idxd
)
325 return test_bit(IDXD_FLAG_PASID_ENABLED
, &idxd
->flags
);
328 static inline bool device_swq_supported(struct idxd_device
*idxd
)
330 return (support_enqcmd
&& device_pasid_enabled(idxd
));
333 enum idxd_portal_prot
{
334 IDXD_PORTAL_UNLIMITED
= 0,
338 enum idxd_interrupt_type
{
343 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot
)
345 return prot
* 0x1000;
348 static inline int idxd_get_wq_portal_full_offset(int wq_id
,
349 enum idxd_portal_prot prot
)
351 return ((wq_id
* 4) << PAGE_SHIFT
) + idxd_get_wq_portal_offset(prot
);
354 static inline void idxd_wq_get(struct idxd_wq
*wq
)
359 static inline void idxd_wq_put(struct idxd_wq
*wq
)
364 static inline int idxd_wq_refcount(struct idxd_wq
*wq
)
366 return wq
->client_count
;
369 int idxd_register_bus_type(void);
370 void idxd_unregister_bus_type(void);
371 int idxd_register_devices(struct idxd_device
*idxd
);
372 void idxd_unregister_devices(struct idxd_device
*idxd
);
373 int idxd_register_driver(void);
374 void idxd_unregister_driver(void);
375 void idxd_wqs_quiesce(struct idxd_device
*idxd
);
377 /* device interrupt control */
378 void idxd_msix_perm_setup(struct idxd_device
*idxd
);
379 void idxd_msix_perm_clear(struct idxd_device
*idxd
);
380 irqreturn_t
idxd_irq_handler(int vec
, void *data
);
381 irqreturn_t
idxd_misc_thread(int vec
, void *data
);
382 irqreturn_t
idxd_wq_thread(int irq
, void *data
);
383 void idxd_mask_error_interrupts(struct idxd_device
*idxd
);
384 void idxd_unmask_error_interrupts(struct idxd_device
*idxd
);
385 void idxd_mask_msix_vectors(struct idxd_device
*idxd
);
386 void idxd_mask_msix_vector(struct idxd_device
*idxd
, int vec_id
);
387 void idxd_unmask_msix_vector(struct idxd_device
*idxd
, int vec_id
);
390 int idxd_device_init_reset(struct idxd_device
*idxd
);
391 int idxd_device_enable(struct idxd_device
*idxd
);
392 int idxd_device_disable(struct idxd_device
*idxd
);
393 void idxd_device_reset(struct idxd_device
*idxd
);
394 void idxd_device_cleanup(struct idxd_device
*idxd
);
395 int idxd_device_config(struct idxd_device
*idxd
);
396 void idxd_device_wqs_clear_state(struct idxd_device
*idxd
);
397 void idxd_device_drain_pasid(struct idxd_device
*idxd
, int pasid
);
398 int idxd_device_load_config(struct idxd_device
*idxd
);
399 int idxd_device_request_int_handle(struct idxd_device
*idxd
, int idx
, int *handle
,
400 enum idxd_interrupt_type irq_type
);
401 int idxd_device_release_int_handle(struct idxd_device
*idxd
, int handle
,
402 enum idxd_interrupt_type irq_type
);
404 /* work queue control */
405 void idxd_wqs_unmap_portal(struct idxd_device
*idxd
);
406 int idxd_wq_alloc_resources(struct idxd_wq
*wq
);
407 void idxd_wq_free_resources(struct idxd_wq
*wq
);
408 int idxd_wq_enable(struct idxd_wq
*wq
);
409 int idxd_wq_disable(struct idxd_wq
*wq
);
410 void idxd_wq_drain(struct idxd_wq
*wq
);
411 void idxd_wq_reset(struct idxd_wq
*wq
);
412 int idxd_wq_map_portal(struct idxd_wq
*wq
);
413 void idxd_wq_unmap_portal(struct idxd_wq
*wq
);
414 void idxd_wq_disable_cleanup(struct idxd_wq
*wq
);
415 int idxd_wq_set_pasid(struct idxd_wq
*wq
, int pasid
);
416 int idxd_wq_disable_pasid(struct idxd_wq
*wq
);
417 void idxd_wq_quiesce(struct idxd_wq
*wq
);
418 int idxd_wq_init_percpu_ref(struct idxd_wq
*wq
);
421 int idxd_submit_desc(struct idxd_wq
*wq
, struct idxd_desc
*desc
);
422 struct idxd_desc
*idxd_alloc_desc(struct idxd_wq
*wq
, enum idxd_op_type optype
);
423 void idxd_free_desc(struct idxd_wq
*wq
, struct idxd_desc
*desc
);
426 int idxd_register_dma_device(struct idxd_device
*idxd
);
427 void idxd_unregister_dma_device(struct idxd_device
*idxd
);
428 int idxd_register_dma_channel(struct idxd_wq
*wq
);
429 void idxd_unregister_dma_channel(struct idxd_wq
*wq
);
430 void idxd_parse_completion_status(u8 status
, enum dmaengine_tx_result
*res
);
431 void idxd_dma_complete_txd(struct idxd_desc
*desc
,
432 enum idxd_complete_type comp_type
);
435 int idxd_cdev_register(void);
436 void idxd_cdev_remove(void);
437 int idxd_cdev_get_major(struct idxd_device
*idxd
);
438 int idxd_wq_add_cdev(struct idxd_wq
*wq
);
439 void idxd_wq_del_cdev(struct idxd_wq
*wq
);