drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_
  25 #define KFD_DEVICE_QUEUE_MANAGER_H_
  26
  27 #include <linux/rwsem.h>
  28 #include <linux/list.h>
  29 #include <linux/mutex.h>
  30 #include <linux/sched/mm.h>
  31 #include "kfd_priv.h"
  32 #include "kfd_mqd_manager.h"
  33
  34
  35 #define VMID_NUM 16
  36
  37 struct device_process_node {
  38         struct qcm_process_device *qpd;
  39         struct list_head list;
  40 };
  41
  42 /**
  43  * struct device_queue_manager_ops
  44  *
  45  * @create_queue: Queue creation routine.
  46  *
  47  * @destroy_queue: Queue destruction routine.
  48  *
  49  * @update_queue: Queue update routine.
  50  *
  51  * @exeute_queues: Dispatches the queues list to the H/W.
  52  *
  53  * @register_process: This routine associates a specific process with device.
  54  *
  55  * @unregister_process: destroys the associations between process to device.
  56  *
  57  * @initialize: Initializes the pipelines and memory module for that device.
  58  *
  59  * @start: Initializes the resources/modules the the device needs for queues
  60  * execution. This function is called on device initialization and after the
  61  * system woke up after suspension.
  62  *
  63  * @stop: This routine stops execution of all the active queue running on the
  64  * H/W and basically this function called on system suspend.
  65  *
  66  * @uninitialize: Destroys all the device queue manager resources allocated in
  67  * initialize routine.
  68  *
  69  * @create_kernel_queue: Creates kernel queue. Used for debug queue.
  70  *
  71  * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
  72  *
  73  * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
  74  * memory apertures.
  75  *
  76  * @process_termination: Clears all process queues belongs to that device.
  77  *
  78  * @evict_process_queues: Evict all active queues of a process
  79  *
  80  * @restore_process_queues: Restore all evicted queues queues of a process
  81  *
  82  * @get_wave_state: Retrieves context save state and optionally copies the
  83  * control stack, if kept in the MQD, to the given userspace address.
  84  */
  85
  86 struct device_queue_manager_ops {
  87         int     (*create_queue)(struct device_queue_manager *dqm,
  88                                 struct queue *q,
  89                                 struct qcm_process_device *qpd);
  90
  91         int     (*destroy_queue)(struct device_queue_manager *dqm,
  92                                 struct qcm_process_device *qpd,
  93                                 struct queue *q);
  94
  95         int     (*update_queue)(struct device_queue_manager *dqm,
  96                                 struct queue *q);
  97
  98         int     (*register_process)(struct device_queue_manager *dqm,
  99                                         struct qcm_process_device *qpd);
 100
 101         int     (*unregister_process)(struct device_queue_manager *dqm,
 102                                         struct qcm_process_device *qpd);
 103
 104         int     (*initialize)(struct device_queue_manager *dqm);
 105         int     (*start)(struct device_queue_manager *dqm);
 106         int     (*stop)(struct device_queue_manager *dqm);
 107         void    (*pre_reset)(struct device_queue_manager *dqm);
 108         void    (*uninitialize)(struct device_queue_manager *dqm);
 109         int     (*create_kernel_queue)(struct device_queue_manager *dqm,
 110                                         struct kernel_queue *kq,
 111                                         struct qcm_process_device *qpd);
 112
 113         void    (*destroy_kernel_queue)(struct device_queue_manager *dqm,
 114                                         struct kernel_queue *kq,
 115                                         struct qcm_process_device *qpd);
 116
 117         bool    (*set_cache_memory_policy)(struct device_queue_manager *dqm,
 118                                            struct qcm_process_device *qpd,
 119                                            enum cache_policy default_policy,
 120                                            enum cache_policy alternate_policy,
 121                                            void __user *alternate_aperture_base,
 122                                            uint64_t alternate_aperture_size);
 123
 124         int     (*set_trap_handler)(struct device_queue_manager *dqm,
 125                                     struct qcm_process_device *qpd,
 126                                     uint64_t tba_addr,
 127                                     uint64_t tma_addr);
 128
 129         int (*process_termination)(struct device_queue_manager *dqm,
 130                         struct qcm_process_device *qpd);
 131
 132         int (*evict_process_queues)(struct device_queue_manager *dqm,
 133                                     struct qcm_process_device *qpd);
 134         int (*restore_process_queues)(struct device_queue_manager *dqm,
 135                                       struct qcm_process_device *qpd);
 136
 137         int     (*get_wave_state)(struct device_queue_manager *dqm,
 138                                   struct queue *q,
 139                                   void __user *ctl_stack,
 140                                   u32 *ctl_stack_used_size,
 141                                   u32 *save_area_used_size);
 142 };
 143
 144 struct device_queue_manager_asic_ops {
 145         int     (*update_qpd)(struct device_queue_manager *dqm,
 146                                         struct qcm_process_device *qpd);
 147         bool    (*set_cache_memory_policy)(struct device_queue_manager *dqm,
 148                                            struct qcm_process_device *qpd,
 149                                            enum cache_policy default_policy,
 150                                            enum cache_policy alternate_policy,
 151                                            void __user *alternate_aperture_base,
 152                                            uint64_t alternate_aperture_size);
 153         void    (*init_sdma_vm)(struct device_queue_manager *dqm,
 154                                 struct queue *q,
 155                                 struct qcm_process_device *qpd);
 156         struct mqd_manager *    (*mqd_manager_init)(enum KFD_MQD_TYPE type,
 157                                  struct kfd_dev *dev);
 158 };
 159
 160 /**
 161  * struct device_queue_manager
 162  *
 163  * This struct is a base class for the kfd queues scheduler in the
 164  * device level. The device base class should expose the basic operations
 165  * for queue creation and queue destruction. This base class hides the
 166  * scheduling mode of the driver and the specific implementation of the
 167  * concrete device. This class is the only class in the queues scheduler
 168  * that configures the H/W.
 169  *
 170  */
 171
 172 struct device_queue_manager {
 173         struct device_queue_manager_ops ops;
 174         struct device_queue_manager_asic_ops asic_ops;
 175
 176         struct mqd_manager      *mqd_mgrs[KFD_MQD_TYPE_MAX];
 177         struct packet_manager   packets;
 178         struct kfd_dev          *dev;
 179         struct mutex            lock_hidden; /* use dqm_lock/unlock(dqm) */
 180         struct list_head        queues;
 181         unsigned int            saved_flags;
 182         unsigned int            processes_count;
 183         unsigned int            active_queue_count;
 184         unsigned int            active_cp_queue_count;
 185         unsigned int            gws_queue_count;
 186         unsigned int            total_queue_count;
 187         unsigned int            next_pipe_to_allocate;
 188         unsigned int            *allocated_queues;
 189         uint64_t                sdma_bitmap;
 190         uint64_t                xgmi_sdma_bitmap;
 191         /* the pasid mapping for each kfd vmid */
 192         uint16_t                vmid_pasid[VMID_NUM];
 193         uint64_t                pipelines_addr;
 194         uint64_t                fence_gpu_addr;
 195         unsigned int            *fence_addr;
 196         struct kfd_mem_obj      *fence_mem;
 197         bool                    active_runlist;
 198         int                     sched_policy;
 199
 200         /* hw exception  */
 201         bool                    is_hws_hang;
 202         bool                    is_resetting;
 203         struct work_struct      hw_exception_work;
 204         struct kfd_mem_obj      hiq_sdma_mqd;
 205         bool                    sched_running;
 206 };
 207
 208 void device_queue_manager_init_cik(
 209                 struct device_queue_manager_asic_ops *asic_ops);
 210 void device_queue_manager_init_cik_hawaii(
 211                 struct device_queue_manager_asic_ops *asic_ops);
 212 void device_queue_manager_init_vi(
 213                 struct device_queue_manager_asic_ops *asic_ops);
 214 void device_queue_manager_init_vi_tonga(
 215                 struct device_queue_manager_asic_ops *asic_ops);
 216 void device_queue_manager_init_v9(
 217                 struct device_queue_manager_asic_ops *asic_ops);
 218 void device_queue_manager_init_v10_navi10(
 219                 struct device_queue_manager_asic_ops *asic_ops);
 220 void program_sh_mem_settings(struct device_queue_manager *dqm,
 221                                         struct qcm_process_device *qpd);
 222 unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
 223 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
 224 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
 225 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
 226 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
 227
 228 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
 229 {
 230         return (pdd->lds_base >> 16) & 0xFF;
 231 }
 232
 233 static inline unsigned int
 234 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
 235 {
 236         return (pdd->lds_base >> 60) & 0x0E;
 237 }
 238
 239 /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
 240  * happens while holding this lock anywhere to prevent deadlocks when
 241  * an MMU notifier runs in reclaim-FS context.
 242  */
 243 static inline void dqm_lock(struct device_queue_manager *dqm)
 244 {
 245         mutex_lock(&dqm->lock_hidden);
 246         dqm->saved_flags = memalloc_nofs_save();
 247 }
 248 static inline void dqm_unlock(struct device_queue_manager *dqm)
 249 {
 250         memalloc_nofs_restore(dqm->saved_flags);
 251         mutex_unlock(&dqm->lock_hidden);
 252 }
 253
 254 static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val)
 255 {
 256         /*
 257          * SDMA activity counter is stored at queue's RPTR + 0x8 location.
 258          */
 259         return get_user(*val, q_rptr + 1);
 260 }
 261 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */