drivers/dma/idxd/idxd.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #ifndef _IDXD_H_
   4 #define _IDXD_H_
   5
   6 #include <linux/sbitmap.h>
   7 #include <linux/dmaengine.h>
   8 #include <linux/percpu-rwsem.h>
   9 #include <linux/wait.h>
  10 #include <linux/cdev.h>
  11 #include <linux/idr.h>
  12 #include "registers.h"
  13
  14 #define IDXD_DRIVER_VERSION     "1.00"
  15
  16 extern struct kmem_cache *idxd_desc_pool;
  17
  18 struct idxd_device;
  19 struct idxd_wq;
  20
  21 #define IDXD_REG_TIMEOUT        50
  22 #define IDXD_DRAIN_TIMEOUT      5000
  23
  24 enum idxd_type {
  25         IDXD_TYPE_UNKNOWN = -1,
  26         IDXD_TYPE_DSA = 0,
  27         IDXD_TYPE_IAX,
  28         IDXD_TYPE_MAX,
  29 };
  30
  31 #define IDXD_NAME_SIZE          128
  32
  33 struct idxd_device_driver {
  34         struct device_driver drv;
  35 };
  36
  37 struct idxd_irq_entry {
  38         struct idxd_device *idxd;
  39         int id;
  40         int vector;
  41         struct llist_head pending_llist;
  42         struct list_head work_list;
  43         /*
  44          * Lock to protect access between irq thread process descriptor
  45          * and irq thread processing error descriptor.
  46          */
  47         spinlock_t list_lock;
  48 };
  49
  50 struct idxd_group {
  51         struct device conf_dev;
  52         struct idxd_device *idxd;
  53         struct grpcfg grpcfg;
  54         int id;
  55         int num_engines;
  56         int num_wqs;
  57         bool use_token_limit;
  58         u8 tokens_allowed;
  59         u8 tokens_reserved;
  60         int tc_a;
  61         int tc_b;
  62 };
  63
  64 #define IDXD_MAX_PRIORITY       0xf
  65
  66 enum idxd_wq_state {
  67         IDXD_WQ_DISABLED = 0,
  68         IDXD_WQ_ENABLED,
  69 };
  70
  71 enum idxd_wq_flag {
  72         WQ_FLAG_DEDICATED = 0,
  73         WQ_FLAG_BLOCK_ON_FAULT,
  74 };
  75
  76 enum idxd_wq_type {
  77         IDXD_WQT_NONE = 0,
  78         IDXD_WQT_KERNEL,
  79         IDXD_WQT_USER,
  80 };
  81
  82 struct idxd_cdev {
  83         struct idxd_wq *wq;
  84         struct cdev cdev;
  85         struct device dev;
  86         int minor;
  87 };
  88
  89 #define IDXD_ALLOCATED_BATCH_SIZE       128U
  90 #define WQ_NAME_SIZE   1024
  91 #define WQ_TYPE_SIZE   10
  92
  93 enum idxd_op_type {
  94         IDXD_OP_BLOCK = 0,
  95         IDXD_OP_NONBLOCK = 1,
  96 };
  97
  98 enum idxd_complete_type {
  99         IDXD_COMPLETE_NORMAL = 0,
 100         IDXD_COMPLETE_ABORT,
 101         IDXD_COMPLETE_DEV_FAIL,
 102 };
 103
 104 struct idxd_dma_chan {
 105         struct dma_chan chan;
 106         struct idxd_wq *wq;
 107 };
 108
 109 struct idxd_wq {
 110         void __iomem *portal;
 111         struct percpu_ref wq_active;
 112         struct completion wq_dead;
 113         struct device conf_dev;
 114         struct idxd_cdev *idxd_cdev;
 115         struct wait_queue_head err_queue;
 116         struct idxd_device *idxd;
 117         int id;
 118         enum idxd_wq_type type;
 119         struct idxd_group *group;
 120         int client_count;
 121         struct mutex wq_lock;   /* mutex for workqueue */
 122         u32 size;
 123         u32 threshold;
 124         u32 priority;
 125         enum idxd_wq_state state;
 126         unsigned long flags;
 127         union wqcfg *wqcfg;
 128         u32 vec_ptr;            /* interrupt steering */
 129         struct dsa_hw_desc **hw_descs;
 130         int num_descs;
 131         union {
 132                 struct dsa_completion_record *compls;
 133                 struct iax_completion_record *iax_compls;
 134         };
 135         void *compls_raw;
 136         dma_addr_t compls_addr;
 137         dma_addr_t compls_addr_raw;
 138         int compls_size;
 139         struct idxd_desc **descs;
 140         struct sbitmap_queue sbq;
 141         struct idxd_dma_chan *idxd_chan;
 142         char name[WQ_NAME_SIZE + 1];
 143         u64 max_xfer_bytes;
 144         u32 max_batch_size;
 145         bool ats_dis;
 146 };
 147
 148 struct idxd_engine {
 149         struct device conf_dev;
 150         int id;
 151         struct idxd_group *group;
 152         struct idxd_device *idxd;
 153 };
 154
 155 /* shadow registers */
 156 struct idxd_hw {
 157         u32 version;
 158         union gen_cap_reg gen_cap;
 159         union wq_cap_reg wq_cap;
 160         union group_cap_reg group_cap;
 161         union engine_cap_reg engine_cap;
 162         struct opcap opcap;
 163         u32 cmd_cap;
 164 };
 165
 166 enum idxd_device_state {
 167         IDXD_DEV_HALTED = -1,
 168         IDXD_DEV_DISABLED = 0,
 169         IDXD_DEV_CONF_READY,
 170         IDXD_DEV_ENABLED,
 171 };
 172
 173 enum idxd_device_flag {
 174         IDXD_FLAG_CONFIGURABLE = 0,
 175         IDXD_FLAG_CMD_RUNNING,
 176         IDXD_FLAG_PASID_ENABLED,
 177 };
 178
 179 struct idxd_dma_dev {
 180         struct idxd_device *idxd;
 181         struct dma_device dma;
 182 };
 183
 184 struct idxd_driver_data {
 185         const char *name_prefix;
 186         enum idxd_type type;
 187         struct device_type *dev_type;
 188         int compl_size;
 189         int align;
 190 };
 191
 192 struct idxd_device {
 193         struct device conf_dev;
 194         struct idxd_driver_data *data;
 195         struct list_head list;
 196         struct idxd_hw hw;
 197         enum idxd_device_state state;
 198         unsigned long flags;
 199         int id;
 200         int major;
 201         u8 cmd_status;
 202
 203         struct pci_dev *pdev;
 204         void __iomem *reg_base;
 205
 206         spinlock_t dev_lock;    /* spinlock for device */
 207         spinlock_t cmd_lock;    /* spinlock for device commands */
 208         struct completion *cmd_done;
 209         struct idxd_group **groups;
 210         struct idxd_wq **wqs;
 211         struct idxd_engine **engines;
 212
 213         struct iommu_sva *sva;
 214         unsigned int pasid;
 215
 216         int num_groups;
 217
 218         u32 msix_perm_offset;
 219         u32 wqcfg_offset;
 220         u32 grpcfg_offset;
 221         u32 perfmon_offset;
 222
 223         u64 max_xfer_bytes;
 224         u32 max_batch_size;
 225         int max_groups;
 226         int max_engines;
 227         int max_tokens;
 228         int max_wqs;
 229         int max_wq_size;
 230         int token_limit;
 231         int nr_tokens;          /* non-reserved tokens */
 232         unsigned int wqcfg_size;
 233
 234         union sw_err_reg sw_err;
 235         wait_queue_head_t cmd_waitq;
 236         int num_wq_irqs;
 237         struct idxd_irq_entry *irq_entries;
 238
 239         struct idxd_dma_dev *idxd_dma;
 240         struct workqueue_struct *wq;
 241         struct work_struct work;
 242
 243         int *int_handles;
 244 };
 245
 246 /* IDXD software descriptor */
 247 struct idxd_desc {
 248         union {
 249                 struct dsa_hw_desc *hw;
 250                 struct iax_hw_desc *iax_hw;
 251         };
 252         dma_addr_t desc_dma;
 253         union {
 254                 struct dsa_completion_record *completion;
 255                 struct iax_completion_record *iax_completion;
 256         };
 257         dma_addr_t compl_dma;
 258         struct dma_async_tx_descriptor txd;
 259         struct llist_node llnode;
 260         struct list_head list;
 261         int id;
 262         int cpu;
 263         unsigned int vector;
 264         struct idxd_wq *wq;
 265 };
 266
 267 #define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
 268 #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
 269
 270 extern struct bus_type dsa_bus_type;
 271 extern struct bus_type iax_bus_type;
 272
 273 extern bool support_enqcmd;
 274 extern struct ida idxd_ida;
 275 extern struct device_type dsa_device_type;
 276 extern struct device_type iax_device_type;
 277 extern struct device_type idxd_wq_device_type;
 278 extern struct device_type idxd_engine_device_type;
 279 extern struct device_type idxd_group_device_type;
 280
 281 static inline bool is_dsa_dev(struct device *dev)
 282 {
 283         return dev->type == &dsa_device_type;
 284 }
 285
 286 static inline bool is_iax_dev(struct device *dev)
 287 {
 288         return dev->type == &iax_device_type;
 289 }
 290
 291 static inline bool is_idxd_dev(struct device *dev)
 292 {
 293         return is_dsa_dev(dev) || is_iax_dev(dev);
 294 }
 295
 296 static inline bool is_idxd_wq_dev(struct device *dev)
 297 {
 298         return dev->type == &idxd_wq_device_type;
 299 }
 300
 301 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
 302 {
 303         if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
 304                 return true;
 305         return false;
 306 }
 307
 308 static inline bool is_idxd_wq_cdev(struct idxd_wq *wq)
 309 {
 310         return wq->type == IDXD_WQT_USER;
 311 }
 312
 313 static inline bool wq_dedicated(struct idxd_wq *wq)
 314 {
 315         return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
 316 }
 317
 318 static inline bool wq_shared(struct idxd_wq *wq)
 319 {
 320         return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
 321 }
 322
 323 static inline bool device_pasid_enabled(struct idxd_device *idxd)
 324 {
 325         return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
 326 }
 327
 328 static inline bool device_swq_supported(struct idxd_device *idxd)
 329 {
 330         return (support_enqcmd && device_pasid_enabled(idxd));
 331 }
 332
 333 enum idxd_portal_prot {
 334         IDXD_PORTAL_UNLIMITED = 0,
 335         IDXD_PORTAL_LIMITED,
 336 };
 337
 338 enum idxd_interrupt_type {
 339         IDXD_IRQ_MSIX = 0,
 340         IDXD_IRQ_IMS,
 341 };
 342
 343 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
 344 {
 345         return prot * 0x1000;
 346 }
 347
 348 static inline int idxd_get_wq_portal_full_offset(int wq_id,
 349                                                  enum idxd_portal_prot prot)
 350 {
 351         return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
 352 }
 353
 354 static inline void idxd_wq_get(struct idxd_wq *wq)
 355 {
 356         wq->client_count++;
 357 }
 358
 359 static inline void idxd_wq_put(struct idxd_wq *wq)
 360 {
 361         wq->client_count--;
 362 }
 363
 364 static inline int idxd_wq_refcount(struct idxd_wq *wq)
 365 {
 366         return wq->client_count;
 367 };
 368
 369 int idxd_register_bus_type(void);
 370 void idxd_unregister_bus_type(void);
 371 int idxd_register_devices(struct idxd_device *idxd);
 372 void idxd_unregister_devices(struct idxd_device *idxd);
 373 int idxd_register_driver(void);
 374 void idxd_unregister_driver(void);
 375 void idxd_wqs_quiesce(struct idxd_device *idxd);
 376
 377 /* device interrupt control */
 378 void idxd_msix_perm_setup(struct idxd_device *idxd);
 379 void idxd_msix_perm_clear(struct idxd_device *idxd);
 380 irqreturn_t idxd_misc_thread(int vec, void *data);
 381 irqreturn_t idxd_wq_thread(int irq, void *data);
 382 void idxd_mask_error_interrupts(struct idxd_device *idxd);
 383 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
 384 void idxd_mask_msix_vectors(struct idxd_device *idxd);
 385 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
 386 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
 387
 388 /* device control */
 389 int idxd_device_init_reset(struct idxd_device *idxd);
 390 int idxd_device_enable(struct idxd_device *idxd);
 391 int idxd_device_disable(struct idxd_device *idxd);
 392 void idxd_device_reset(struct idxd_device *idxd);
 393 void idxd_device_cleanup(struct idxd_device *idxd);
 394 int idxd_device_config(struct idxd_device *idxd);
 395 void idxd_device_wqs_clear_state(struct idxd_device *idxd);
 396 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
 397 int idxd_device_load_config(struct idxd_device *idxd);
 398 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
 399                                    enum idxd_interrupt_type irq_type);
 400 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
 401                                    enum idxd_interrupt_type irq_type);
 402
 403 /* work queue control */
 404 void idxd_wqs_unmap_portal(struct idxd_device *idxd);
 405 int idxd_wq_alloc_resources(struct idxd_wq *wq);
 406 void idxd_wq_free_resources(struct idxd_wq *wq);
 407 int idxd_wq_enable(struct idxd_wq *wq);
 408 int idxd_wq_disable(struct idxd_wq *wq);
 409 void idxd_wq_drain(struct idxd_wq *wq);
 410 void idxd_wq_reset(struct idxd_wq *wq);
 411 int idxd_wq_map_portal(struct idxd_wq *wq);
 412 void idxd_wq_unmap_portal(struct idxd_wq *wq);
 413 void idxd_wq_disable_cleanup(struct idxd_wq *wq);
 414 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
 415 int idxd_wq_disable_pasid(struct idxd_wq *wq);
 416 void idxd_wq_quiesce(struct idxd_wq *wq);
 417 int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
 418
 419 /* submission */
 420 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
 421 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
 422 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
 423
 424 /* dmaengine */
 425 int idxd_register_dma_device(struct idxd_device *idxd);
 426 void idxd_unregister_dma_device(struct idxd_device *idxd);
 427 int idxd_register_dma_channel(struct idxd_wq *wq);
 428 void idxd_unregister_dma_channel(struct idxd_wq *wq);
 429 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
 430 void idxd_dma_complete_txd(struct idxd_desc *desc,
 431                            enum idxd_complete_type comp_type);
 432
 433 /* cdev */
 434 int idxd_cdev_register(void);
 435 void idxd_cdev_remove(void);
 436 int idxd_cdev_get_major(struct idxd_device *idxd);
 437 int idxd_wq_add_cdev(struct idxd_wq *wq);
 438 void idxd_wq_del_cdev(struct idxd_wq *wq);
 439
 440 #endif