drivers/gpu/drm/msm/adreno/a6xx_hfi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
   3
   4 #include <linux/completion.h>
   5 #include <linux/circ_buf.h>
   6 #include <linux/list.h>
   7
   8 #include "a6xx_gmu.h"
   9 #include "a6xx_gmu.xml.h"
  10 #include "a6xx_gpu.h"
  11
  12 #define HFI_MSG_ID(val) [val] = #val
  13
  14 static const char * const a6xx_hfi_msg_id[] = {
  15         HFI_MSG_ID(HFI_H2F_MSG_INIT),
  16         HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION),
  17         HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE),
  18         HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE),
  19         HFI_MSG_ID(HFI_H2F_MSG_TEST),
  20 };
  21
  22 static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data,
  23                 u32 dwords)
  24 {
  25         struct a6xx_hfi_queue_header *header = queue->header;
  26         u32 i, hdr, index = header->read_index;
  27
  28         if (header->read_index == header->write_index) {
  29                 header->rx_request = 1;
  30                 return 0;
  31         }
  32
  33         hdr = queue->data[index];
  34
  35         /*
  36          * If we are to assume that the GMU firmware is in fact a rational actor
  37          * and is programmed to not send us a larger response than we expect
  38          * then we can also assume that if the header size is unexpectedly large
  39          * that it is due to memory corruption and/or hardware failure. In this
  40          * case the only reasonable course of action is to BUG() to help harden
  41          * the failure.
  42          */
  43
  44         BUG_ON(HFI_HEADER_SIZE(hdr) > dwords);
  45
  46         for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) {
  47                 data[i] = queue->data[index];
  48                 index = (index + 1) % header->size;
  49         }
  50
  51         header->read_index = index;
  52         return HFI_HEADER_SIZE(hdr);
  53 }
  54
  55 static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu,
  56         struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
  57 {
  58         struct a6xx_hfi_queue_header *header = queue->header;
  59         u32 i, space, index = header->write_index;
  60
  61         spin_lock(&queue->lock);
  62
  63         space = CIRC_SPACE(header->write_index, header->read_index,
  64                 header->size);
  65         if (space < dwords) {
  66                 header->dropped++;
  67                 spin_unlock(&queue->lock);
  68                 return -ENOSPC;
  69         }
  70
  71         for (i = 0; i < dwords; i++) {
  72                 queue->data[index] = data[i];
  73                 index = (index + 1) % header->size;
  74         }
  75
  76         header->write_index = index;
  77         spin_unlock(&queue->lock);
  78
  79         gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01);
  80         return 0;
  81 }
  82
  83 static int a6xx_hfi_wait_for_ack(struct a6xx_gmu *gmu, u32 id, u32 seqnum,
  84                 u32 *payload, u32 payload_size)
  85 {
  86         struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE];
  87         u32 val;
  88         int ret;
  89
  90         /* Wait for a response */
  91         ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
  92                 val & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 100, 5000);
  93
  94         if (ret) {
  95                 DRM_DEV_ERROR(gmu->dev,
  96                         "Message %s id %d timed out waiting for response\n",
  97                         a6xx_hfi_msg_id[id], seqnum);
  98                 return -ETIMEDOUT;
  99         }
 100
 101         /* Clear the interrupt */
 102         gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR,
 103                 A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ);
 104
 105         for (;;) {
 106                 struct a6xx_hfi_msg_response resp;
 107
 108                 /* Get the next packet */
 109                 ret = a6xx_hfi_queue_read(queue, (u32 *) &resp,
 110                         sizeof(resp) >> 2);
 111
 112                 /* If the queue is empty our response never made it */
 113                 if (!ret) {
 114                         DRM_DEV_ERROR(gmu->dev,
 115                                 "The HFI response queue is unexpectedly empty\n");
 116
 117                         return -ENOENT;
 118                 }
 119
 120                 if (HFI_HEADER_ID(resp.header) == HFI_F2H_MSG_ERROR) {
 121                         struct a6xx_hfi_msg_error *error =
 122                                 (struct a6xx_hfi_msg_error *) &resp;
 123
 124                         DRM_DEV_ERROR(gmu->dev, "GMU firmware error %d\n",
 125                                 error->code);
 126                         continue;
 127                 }
 128
 129                 if (seqnum != HFI_HEADER_SEQNUM(resp.ret_header)) {
 130                         DRM_DEV_ERROR(gmu->dev,
 131                                 "Unexpected message id %d on the response queue\n",
 132                                 HFI_HEADER_SEQNUM(resp.ret_header));
 133                         continue;
 134                 }
 135
 136                 if (resp.error) {
 137                         DRM_DEV_ERROR(gmu->dev,
 138                                 "Message %s id %d returned error %d\n",
 139                                 a6xx_hfi_msg_id[id], seqnum, resp.error);
 140                         return -EINVAL;
 141                 }
 142
 143                 /* All is well, copy over the buffer */
 144                 if (payload && payload_size)
 145                         memcpy(payload, resp.payload,
 146                                 min_t(u32, payload_size, sizeof(resp.payload)));
 147
 148                 return 0;
 149         }
 150 }
 151
 152 static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id,
 153                 void *data, u32 size, u32 *payload, u32 payload_size)
 154 {
 155         struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE];
 156         int ret, dwords = size >> 2;
 157         u32 seqnum;
 158
 159         seqnum = atomic_inc_return(&queue->seqnum) % 0xfff;
 160
 161         /* First dword of the message is the message header - fill it in */
 162         *((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) |
 163                 (dwords << 8) | id;
 164
 165         ret = a6xx_hfi_queue_write(gmu, queue, data, dwords);
 166         if (ret) {
 167                 DRM_DEV_ERROR(gmu->dev, "Unable to send message %s id %d\n",
 168                         a6xx_hfi_msg_id[id], seqnum);
 169                 return ret;
 170         }
 171
 172         return a6xx_hfi_wait_for_ack(gmu, id, seqnum, payload, payload_size);
 173 }
 174
 175 static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state)
 176 {
 177         struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 };
 178
 179         msg.dbg_buffer_addr = (u32) gmu->debug->iova;
 180         msg.dbg_buffer_size = (u32) gmu->debug->size;
 181         msg.boot_state = boot_state;
 182
 183         return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg),
 184                 NULL, 0);
 185 }
 186
 187 static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version)
 188 {
 189         struct a6xx_hfi_msg_fw_version msg = { 0 };
 190
 191         /* Currently supporting version 1.1 */
 192         msg.supported_version = (1 << 28) | (1 << 16);
 193
 194         return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg),
 195                 version, sizeof(*version));
 196 }
 197
 198 static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
 199 {
 200         struct a6xx_hfi_msg_perf_table msg = { 0 };
 201         int i;
 202
 203         msg.num_gpu_levels = gmu->nr_gpu_freqs;
 204         msg.num_gmu_levels = gmu->nr_gmu_freqs;
 205
 206         for (i = 0; i < gmu->nr_gpu_freqs; i++) {
 207                 msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
 208                 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
 209         }
 210
 211         for (i = 0; i < gmu->nr_gmu_freqs; i++) {
 212                 msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
 213                 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
 214         }
 215
 216         return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
 217                 NULL, 0);
 218 }
 219
 220 static void a618_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 221 {
 222         /* Send a single "off" entry since the 618 GMU doesn't do bus scaling */
 223         msg->bw_level_num = 1;
 224
 225         msg->ddr_cmds_num = 3;
 226         msg->ddr_wait_bitmask = 0x01;
 227
 228         msg->ddr_cmds_addrs[0] = 0x50000;
 229         msg->ddr_cmds_addrs[1] = 0x5003c;
 230         msg->ddr_cmds_addrs[2] = 0x5000c;
 231
 232         msg->ddr_cmds_data[0][0] =  0x40000000;
 233         msg->ddr_cmds_data[0][1] =  0x40000000;
 234         msg->ddr_cmds_data[0][2] =  0x40000000;
 235
 236         /*
 237          * These are the CX (CNOC) votes - these are used by the GMU but the
 238          * votes are known and fixed for the target
 239          */
 240         msg->cnoc_cmds_num = 1;
 241         msg->cnoc_wait_bitmask = 0x01;
 242
 243         msg->cnoc_cmds_addrs[0] = 0x5007c;
 244         msg->cnoc_cmds_data[0][0] =  0x40000000;
 245         msg->cnoc_cmds_data[1][0] =  0x60000001;
 246 }
 247
 248 static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
 249 {
 250         /* Send a single "off" entry since the 630 GMU doesn't do bus scaling */
 251         msg->bw_level_num = 1;
 252
 253         msg->ddr_cmds_num = 3;
 254         msg->ddr_wait_bitmask = 0x07;
 255
 256         msg->ddr_cmds_addrs[0] = 0x50000;
 257         msg->ddr_cmds_addrs[1] = 0x5005c;
 258         msg->ddr_cmds_addrs[2] = 0x5000c;
 259
 260         msg->ddr_cmds_data[0][0] =  0x40000000;
 261         msg->ddr_cmds_data[0][1] =  0x40000000;
 262         msg->ddr_cmds_data[0][2] =  0x40000000;
 263
 264         /*
 265          * These are the CX (CNOC) votes.  This is used but the values for the
 266          * sdm845 GMU are known and fixed so we can hard code them.
 267          */
 268
 269         msg->cnoc_cmds_num = 3;
 270         msg->cnoc_wait_bitmask = 0x05;
 271
 272         msg->cnoc_cmds_addrs[0] = 0x50034;
 273         msg->cnoc_cmds_addrs[1] = 0x5007c;
 274         msg->cnoc_cmds_addrs[2] = 0x5004c;
 275
 276         msg->cnoc_cmds_data[0][0] =  0x40000000;
 277         msg->cnoc_cmds_data[0][1] =  0x00000000;
 278         msg->cnoc_cmds_data[0][2] =  0x40000000;
 279
 280         msg->cnoc_cmds_data[1][0] =  0x60000001;
 281         msg->cnoc_cmds_data[1][1] =  0x20000001;
 282         msg->cnoc_cmds_data[1][2] =  0x60000001;
 283 }
 284
 285
 286 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
 287 {
 288         struct a6xx_hfi_msg_bw_table msg = { 0 };
 289         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 290         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 291
 292         if (adreno_is_a618(adreno_gpu))
 293                 a618_build_bw_table(&msg);
 294         else
 295                 a6xx_build_bw_table(&msg);
 296
 297         return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg),
 298                 NULL, 0);
 299 }
 300
 301 static int a6xx_hfi_send_test(struct a6xx_gmu *gmu)
 302 {
 303         struct a6xx_hfi_msg_test msg = { 0 };
 304
 305         return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg),
 306                 NULL, 0);
 307 }
 308
 309 int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
 310 {
 311         int ret;
 312
 313         ret = a6xx_hfi_send_gmu_init(gmu, boot_state);
 314         if (ret)
 315                 return ret;
 316
 317         ret = a6xx_hfi_get_fw_version(gmu, NULL);
 318         if (ret)
 319                 return ret;
 320
 321         /*
 322          * We have to get exchange version numbers per the sequence but at this
 323          * point th kernel driver doesn't need to know the exact version of
 324          * the GMU firmware
 325          */
 326
 327         ret = a6xx_hfi_send_perf_table(gmu);
 328         if (ret)
 329                 return ret;
 330
 331         ret = a6xx_hfi_send_bw_table(gmu);
 332         if (ret)
 333                 return ret;
 334
 335         /*
 336          * Let the GMU know that there won't be any more HFI messages until next
 337          * boot
 338          */
 339         a6xx_hfi_send_test(gmu);
 340
 341         return 0;
 342 }
 343
 344 void a6xx_hfi_stop(struct a6xx_gmu *gmu)
 345 {
 346         int i;
 347
 348         for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) {
 349                 struct a6xx_hfi_queue *queue = &gmu->queues[i];
 350
 351                 if (!queue->header)
 352                         continue;
 353
 354                 if (queue->header->read_index != queue->header->write_index)
 355                         DRM_DEV_ERROR(gmu->dev, "HFI queue %d is not empty\n", i);
 356
 357                 queue->header->read_index = 0;
 358                 queue->header->write_index = 0;
 359         }
 360 }
 361
 362 static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue,
 363                 struct a6xx_hfi_queue_header *header, void *virt, u64 iova,
 364                 u32 id)
 365 {
 366         spin_lock_init(&queue->lock);
 367         queue->header = header;
 368         queue->data = virt;
 369         atomic_set(&queue->seqnum, 0);
 370
 371         /* Set up the shared memory header */
 372         header->iova = iova;
 373         header->type =  10 << 8 | id;
 374         header->status = 1;
 375         header->size = SZ_4K >> 2;
 376         header->msg_size = 0;
 377         header->dropped = 0;
 378         header->rx_watermark = 1;
 379         header->tx_watermark = 1;
 380         header->rx_request = 1;
 381         header->tx_request = 0;
 382         header->read_index = 0;
 383         header->write_index = 0;
 384 }
 385
 386 void a6xx_hfi_init(struct a6xx_gmu *gmu)
 387 {
 388         struct a6xx_gmu_bo *hfi = gmu->hfi;
 389         struct a6xx_hfi_queue_table_header *table = hfi->virt;
 390         struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table);
 391         u64 offset;
 392         int table_size;
 393
 394         /*
 395          * The table size is the size of the table header plus all of the queue
 396          * headers
 397          */
 398         table_size = sizeof(*table);
 399         table_size += (ARRAY_SIZE(gmu->queues) *
 400                 sizeof(struct a6xx_hfi_queue_header));
 401
 402         table->version = 0;
 403         table->size = table_size;
 404         /* First queue header is located immediately after the table header */
 405         table->qhdr0_offset = sizeof(*table) >> 2;
 406         table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2;
 407         table->num_queues = ARRAY_SIZE(gmu->queues);
 408         table->active_queues = ARRAY_SIZE(gmu->queues);
 409
 410         /* Command queue */
 411         offset = SZ_4K;
 412         a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset,
 413                 hfi->iova + offset, 0);
 414
 415         /* GMU response queue */
 416         offset += SZ_4K;
 417         a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset,
 418                 hfi->iova + offset, 4);
 419 }