drivers/gpu/drm/msm/adreno/a5xx_gpu.c

   1 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License version 2 and
   5  * only version 2 as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  */
  13
  14 #include <linux/types.h>
  15 #include <linux/cpumask.h>
  16 #include <linux/qcom_scm.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/of_reserved_mem.h>
  19 #include <linux/soc/qcom/mdt_loader.h>
  20 #include "msm_gem.h"
  21 #include "msm_mmu.h"
  22 #include "a5xx_gpu.h"
  23
  24 extern bool hang_debug;
  25 static void a5xx_dump(struct msm_gpu *gpu);
  26
  27 #define GPU_PAS_ID 13
  28
  29 #if IS_ENABLED(CONFIG_QCOM_MDT_LOADER)
  30
  31 static int zap_shader_load_mdt(struct device *dev, const char *fwname)
  32 {
  33         const struct firmware *fw;
  34         phys_addr_t mem_phys;
  35         ssize_t mem_size;
  36         void *mem_region = NULL;
  37         int ret;
  38
  39         /* Request the MDT file for the firmware */
  40         ret = request_firmware(&fw, fwname, dev);
  41         if (ret) {
  42                 DRM_DEV_ERROR(dev, "Unable to load %s\n", fwname);
  43                 return ret;
  44         }
  45
  46         /* Figure out how much memory we need */
  47         mem_size = qcom_mdt_get_size(fw);
  48         if (mem_size < 0) {
  49                 ret = mem_size;
  50                 goto out;
  51         }
  52
  53         /* Allocate memory for the firmware image */
  54         mem_region = dmam_alloc_coherent(dev, mem_size, &mem_phys, GFP_KERNEL);
  55         if (!mem_region) {
  56                 ret = -ENOMEM;
  57                 goto out;
  58         }
  59
  60         /* Load the rest of the MDT */
  61         ret = qcom_mdt_load(dev, fw, fwname, GPU_PAS_ID, mem_region, mem_phys,
  62                 mem_size);
  63         if (ret)
  64                 goto out;
  65
  66         /* Send the image to the secure world */
  67         ret = qcom_scm_pas_auth_and_reset(GPU_PAS_ID);
  68         if (ret)
  69                 DRM_DEV_ERROR(dev, "Unable to authorize the image\n");
  70
  71 out:
  72         release_firmware(fw);
  73
  74         return ret;
  75 }
  76 #else
  77 static int zap_shader_load_mdt(struct device *dev, const char *fwname)
  78 {
  79         return -ENODEV;
  80 }
  81 #endif
  82
  83 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
  84         struct msm_file_private *ctx)
  85 {
  86         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
  87         struct msm_drm_private *priv = gpu->dev->dev_private;
  88         struct msm_ringbuffer *ring = gpu->rb;
  89         unsigned int i, ibs = 0;
  90
  91         for (i = 0; i < submit->nr_cmds; i++) {
  92                 switch (submit->cmd[i].type) {
  93                 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
  94                         break;
  95                 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
  96                         if (priv->lastctx == ctx)
  97                                 break;
  98                 case MSM_SUBMIT_CMD_BUF:
  99                         OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
 100                         OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
 101                         OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
 102                         OUT_RING(ring, submit->cmd[i].size);
 103                         ibs++;
 104                         break;
 105                 }
 106         }
 107
 108         OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
 109         OUT_RING(ring, submit->fence->seqno);
 110
 111         OUT_PKT7(ring, CP_EVENT_WRITE, 4);
 112         OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
 113         OUT_RING(ring, lower_32_bits(rbmemptr(adreno_gpu, fence)));
 114         OUT_RING(ring, upper_32_bits(rbmemptr(adreno_gpu, fence)));
 115         OUT_RING(ring, submit->fence->seqno);
 116
 117         gpu->funcs->flush(gpu);
 118 }
 119
 120 struct a5xx_hwcg {
 121         u32 offset;
 122         u32 value;
 123 };
 124
 125 static const struct a5xx_hwcg a530_hwcg[] = {
 126         {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 127         {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
 128         {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
 129         {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
 130         {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 131         {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
 132         {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
 133         {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
 134         {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 135         {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
 136         {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
 137         {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
 138         {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 139         {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
 140         {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
 141         {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
 142         {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
 143         {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
 144         {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
 145         {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
 146         {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 147         {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
 148         {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
 149         {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
 150         {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
 151         {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
 152         {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
 153         {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
 154         {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 155         {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
 156         {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
 157         {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
 158         {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 159         {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
 160         {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
 161         {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
 162         {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
 163         {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
 164         {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
 165         {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
 166         {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 167         {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
 168         {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
 169         {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
 170         {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 171         {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
 172         {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
 173         {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
 174         {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
 175         {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
 176         {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
 177         {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
 178         {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 179         {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
 180         {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
 181         {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
 182         {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
 183         {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 184         {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 185         {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
 186         {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
 187         {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
 188         {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
 189         {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
 190         {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
 191         {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
 192         {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
 193         {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
 194         {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
 195         {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
 196         {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
 197         {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
 198         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
 199         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
 200         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
 201         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
 202         {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
 203         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
 204         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
 205         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
 206         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
 207         {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
 208         {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 209         {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
 210         {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 211         {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 212         {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 213         {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 214         {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 215         {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 216         {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 217         {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
 218 };
 219
 220 static const struct {
 221         int (*test)(struct adreno_gpu *gpu);
 222         const struct a5xx_hwcg *regs;
 223         unsigned int count;
 224 } a5xx_hwcg_regs[] = {
 225         { adreno_is_a530, a530_hwcg, ARRAY_SIZE(a530_hwcg), },
 226 };
 227
 228 static void _a5xx_enable_hwcg(struct msm_gpu *gpu,
 229                 const struct a5xx_hwcg *regs, unsigned int count)
 230 {
 231         unsigned int i;
 232
 233         for (i = 0; i < count; i++)
 234                 gpu_write(gpu, regs[i].offset, regs[i].value);
 235
 236         gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xAAA8AA00);
 237         gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, 0x182);
 238 }
 239
 240 static void a5xx_enable_hwcg(struct msm_gpu *gpu)
 241 {
 242         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 243         unsigned int i;
 244
 245         for (i = 0; i < ARRAY_SIZE(a5xx_hwcg_regs); i++) {
 246                 if (a5xx_hwcg_regs[i].test(adreno_gpu)) {
 247                         _a5xx_enable_hwcg(gpu, a5xx_hwcg_regs[i].regs,
 248                                 a5xx_hwcg_regs[i].count);
 249                         return;
 250                 }
 251         }
 252 }
 253
 254 static int a5xx_me_init(struct msm_gpu *gpu)
 255 {
 256         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 257         struct msm_ringbuffer *ring = gpu->rb;
 258
 259         OUT_PKT7(ring, CP_ME_INIT, 8);
 260
 261         OUT_RING(ring, 0x0000002F);
 262
 263         /* Enable multiple hardware contexts */
 264         OUT_RING(ring, 0x00000003);
 265
 266         /* Enable error detection */
 267         OUT_RING(ring, 0x20000000);
 268
 269         /* Don't enable header dump */
 270         OUT_RING(ring, 0x00000000);
 271         OUT_RING(ring, 0x00000000);
 272
 273         /* Specify workarounds for various microcode issues */
 274         if (adreno_is_a530(adreno_gpu)) {
 275                 /* Workaround for token end syncs
 276                  * Force a WFI after every direct-render 3D mode draw and every
 277                  * 2D mode 3 draw
 278                  */
 279                 OUT_RING(ring, 0x0000000B);
 280         } else {
 281                 /* No workarounds enabled */
 282                 OUT_RING(ring, 0x00000000);
 283         }
 284
 285         OUT_RING(ring, 0x00000000);
 286         OUT_RING(ring, 0x00000000);
 287
 288         gpu->funcs->flush(gpu);
 289
 290         return a5xx_idle(gpu) ? 0 : -EINVAL;
 291 }
 292
 293 static struct drm_gem_object *a5xx_ucode_load_bo(struct msm_gpu *gpu,
 294                 const struct firmware *fw, u64 *iova)
 295 {
 296         struct drm_device *drm = gpu->dev;
 297         struct drm_gem_object *bo;
 298         void *ptr;
 299
 300         bo = msm_gem_new_locked(drm, fw->size - 4, MSM_BO_UNCACHED);
 301         if (IS_ERR(bo))
 302                 return bo;
 303
 304         ptr = msm_gem_get_vaddr(bo);
 305         if (!ptr) {
 306                 drm_gem_object_unreference(bo);
 307                 return ERR_PTR(-ENOMEM);
 308         }
 309
 310         if (iova) {
 311                 int ret = msm_gem_get_iova(bo, gpu->aspace, iova);
 312
 313                 if (ret) {
 314                         drm_gem_object_unreference(bo);
 315                         return ERR_PTR(ret);
 316                 }
 317         }
 318
 319         memcpy(ptr, &fw->data[4], fw->size - 4);
 320
 321         msm_gem_put_vaddr(bo);
 322         return bo;
 323 }
 324
 325 static int a5xx_ucode_init(struct msm_gpu *gpu)
 326 {
 327         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 328         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 329         int ret;
 330
 331         if (!a5xx_gpu->pm4_bo) {
 332                 a5xx_gpu->pm4_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pm4,
 333                         &a5xx_gpu->pm4_iova);
 334
 335                 if (IS_ERR(a5xx_gpu->pm4_bo)) {
 336                         ret = PTR_ERR(a5xx_gpu->pm4_bo);
 337                         a5xx_gpu->pm4_bo = NULL;
 338                         dev_err(gpu->dev->dev, "could not allocate PM4: %d\n",
 339                                 ret);
 340                         return ret;
 341                 }
 342         }
 343
 344         if (!a5xx_gpu->pfp_bo) {
 345                 a5xx_gpu->pfp_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pfp,
 346                         &a5xx_gpu->pfp_iova);
 347
 348                 if (IS_ERR(a5xx_gpu->pfp_bo)) {
 349                         ret = PTR_ERR(a5xx_gpu->pfp_bo);
 350                         a5xx_gpu->pfp_bo = NULL;
 351                         dev_err(gpu->dev->dev, "could not allocate PFP: %d\n",
 352                                 ret);
 353                         return ret;
 354                 }
 355         }
 356
 357         gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,
 358                 REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova);
 359
 360         gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,
 361                 REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova);
 362
 363         return 0;
 364 }
 365
 366 #define SCM_GPU_ZAP_SHADER_RESUME 0
 367
 368 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
 369 {
 370         int ret;
 371
 372         ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
 373         if (ret)
 374                 DRM_ERROR("%s: zap-shader resume failed: %d\n",
 375                         gpu->name, ret);
 376
 377         return ret;
 378 }
 379
 380 /* Set up a child device to "own" the zap shader */
 381 static int a5xx_zap_shader_dev_init(struct device *parent, struct device *dev)
 382 {
 383         struct device_node *node;
 384         int ret;
 385
 386         if (dev->parent)
 387                 return 0;
 388
 389         /* Find the sub-node for the zap shader */
 390         node = of_get_child_by_name(parent->of_node, "zap-shader");
 391         if (!node) {
 392                 DRM_DEV_ERROR(parent, "zap-shader not found in device tree\n");
 393                 return -ENODEV;
 394         }
 395
 396         dev->parent = parent;
 397         dev->of_node = node;
 398         dev_set_name(dev, "adreno_zap_shader");
 399
 400         ret = device_register(dev);
 401         if (ret) {
 402                 DRM_DEV_ERROR(parent, "Couldn't register zap shader device\n");
 403                 goto out;
 404         }
 405
 406         ret = of_reserved_mem_device_init(dev);
 407         if (ret) {
 408                 DRM_DEV_ERROR(parent, "Unable to set up the reserved memory\n");
 409                 device_unregister(dev);
 410         }
 411
 412 out:
 413         if (ret)
 414                 dev->parent = NULL;
 415
 416         return ret;
 417 }
 418
 419 static int a5xx_zap_shader_init(struct msm_gpu *gpu)
 420 {
 421         static bool loaded;
 422         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 423         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 424         struct platform_device *pdev = a5xx_gpu->pdev;
 425         int ret;
 426
 427         /*
 428          * If the zap shader is already loaded into memory we just need to kick
 429          * the remote processor to reinitialize it
 430          */
 431         if (loaded)
 432                 return a5xx_zap_shader_resume(gpu);
 433
 434         /* We need SCM to be able to load the firmware */
 435         if (!qcom_scm_is_available()) {
 436                 DRM_DEV_ERROR(&pdev->dev, "SCM is not available\n");
 437                 return -EPROBE_DEFER;
 438         }
 439
 440         /* Each GPU has a target specific zap shader firmware name to use */
 441         if (!adreno_gpu->info->zapfw) {
 442                 DRM_DEV_ERROR(&pdev->dev,
 443                         "Zap shader firmware file not specified for this target\n");
 444                 return -ENODEV;
 445         }
 446
 447         ret = a5xx_zap_shader_dev_init(&pdev->dev, &a5xx_gpu->zap_dev);
 448
 449         if (!ret)
 450                 ret = zap_shader_load_mdt(&a5xx_gpu->zap_dev,
 451                         adreno_gpu->info->zapfw);
 452
 453         loaded = !ret;
 454
 455         return ret;
 456 }
 457
 458 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
 459           A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
 460           A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
 461           A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
 462           A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
 463           A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
 464           A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
 465           A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
 466           A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
 467           A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
 468
 469 static int a5xx_hw_init(struct msm_gpu *gpu)
 470 {
 471         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 472         int ret;
 473
 474         gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
 475
 476         /* Make all blocks contribute to the GPU BUSY perf counter */
 477         gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
 478
 479         /* Enable RBBM error reporting bits */
 480         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
 481
 482         if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
 483                 /*
 484                  * Mask out the activity signals from RB1-3 to avoid false
 485                  * positives
 486                  */
 487
 488                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
 489                         0xF0000000);
 490                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
 491                         0xFFFFFFFF);
 492                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
 493                         0xFFFFFFFF);
 494                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
 495                         0xFFFFFFFF);
 496                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
 497                         0xFFFFFFFF);
 498                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
 499                         0xFFFFFFFF);
 500                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
 501                         0xFFFFFFFF);
 502                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
 503                         0xFFFFFFFF);
 504         }
 505
 506         /* Enable fault detection */
 507         gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
 508                 (1 << 30) | 0xFFFF);
 509
 510         /* Turn on performance counters */
 511         gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
 512
 513         /* Increase VFD cache access so LRZ and other data gets evicted less */
 514         gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
 515
 516         /* Disable L2 bypass in the UCHE */
 517         gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
 518         gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
 519         gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
 520         gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
 521
 522         /* Set the GMEM VA range (0 to gpu->gmem) */
 523         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
 524         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
 525         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
 526                 0x00100000 + adreno_gpu->gmem - 1);
 527         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
 528
 529         gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
 530         gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
 531         gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
 532         gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
 533
 534         gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22));
 535
 536         if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
 537                 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
 538
 539         gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100);
 540
 541         /* Enable USE_RETENTION_FLOPS */
 542         gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
 543
 544         /* Enable ME/PFP split notification */
 545         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
 546
 547         /* Enable HWCG */
 548         a5xx_enable_hwcg(gpu);
 549
 550         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
 551
 552         /* Set the highest bank bit */
 553         gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7);
 554         gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1);
 555
 556         /* Protect registers from the CP */
 557         gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
 558
 559         /* RBBM */
 560         gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
 561         gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
 562         gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
 563         gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
 564         gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
 565         gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
 566
 567         /* Content protect */
 568         gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
 569                 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
 570                         16));
 571         gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
 572                 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
 573
 574         /* CP */
 575         gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
 576         gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
 577         gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
 578         gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
 579
 580         /* RB */
 581         gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
 582         gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
 583
 584         /* VPC */
 585         gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
 586         gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4));
 587
 588         /* UCHE */
 589         gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
 590
 591         if (adreno_is_a530(adreno_gpu))
 592                 gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
 593                         ADRENO_PROTECT_RW(0x10000, 0x8000));
 594
 595         gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
 596         /*
 597          * Disable the trusted memory range - we don't actually supported secure
 598          * memory rendering at this point in time and we don't want to block off
 599          * part of the virtual memory space.
 600          */
 601         gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
 602                 REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
 603         gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
 604
 605         /* Load the GPMU firmware before starting the HW init */
 606         a5xx_gpmu_ucode_init(gpu);
 607
 608         ret = adreno_hw_init(gpu);
 609         if (ret)
 610                 return ret;
 611
 612         ret = a5xx_ucode_init(gpu);
 613         if (ret)
 614                 return ret;
 615
 616         /* Disable the interrupts through the initial bringup stage */
 617         gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
 618
 619         /* Clear ME_HALT to start the micro engine */
 620         gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
 621         ret = a5xx_me_init(gpu);
 622         if (ret)
 623                 return ret;
 624
 625         ret = a5xx_power_init(gpu);
 626         if (ret)
 627                 return ret;
 628
 629         /*
 630          * Send a pipeline event stat to get misbehaving counters to start
 631          * ticking correctly
 632          */
 633         if (adreno_is_a530(adreno_gpu)) {
 634                 OUT_PKT7(gpu->rb, CP_EVENT_WRITE, 1);
 635                 OUT_RING(gpu->rb, 0x0F);
 636
 637                 gpu->funcs->flush(gpu);
 638                 if (!a5xx_idle(gpu))
 639                         return -EINVAL;
 640         }
 641
 642         /*
 643          * Try to load a zap shader into the secure world. If successful
 644          * we can use the CP to switch out of secure mode. If not then we
 645          * have no resource but to try to switch ourselves out manually. If we
 646          * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
 647          * be blocked and a permissions violation will soon follow.
 648          */
 649         ret = a5xx_zap_shader_init(gpu);
 650         if (!ret) {
 651                 OUT_PKT7(gpu->rb, CP_SET_SECURE_MODE, 1);
 652                 OUT_RING(gpu->rb, 0x00000000);
 653
 654                 gpu->funcs->flush(gpu);
 655                 if (!a5xx_idle(gpu))
 656                         return -EINVAL;
 657         } else {
 658                 /* Print a warning so if we die, we know why */
 659                 dev_warn_once(gpu->dev->dev,
 660                         "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
 661                 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
 662         }
 663
 664         return 0;
 665 }
 666
 667 static void a5xx_recover(struct msm_gpu *gpu)
 668 {
 669         int i;
 670
 671         adreno_dump_info(gpu);
 672
 673         for (i = 0; i < 8; i++) {
 674                 printk("CP_SCRATCH_REG%d: %u\n", i,
 675                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
 676         }
 677
 678         if (hang_debug)
 679                 a5xx_dump(gpu);
 680
 681         gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
 682         gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
 683         gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
 684         adreno_recover(gpu);
 685 }
 686
 687 static void a5xx_destroy(struct msm_gpu *gpu)
 688 {
 689         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 690         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 691
 692         DBG("%s", gpu->name);
 693
 694         if (a5xx_gpu->zap_dev.parent)
 695                 device_unregister(&a5xx_gpu->zap_dev);
 696
 697         if (a5xx_gpu->pm4_bo) {
 698                 if (a5xx_gpu->pm4_iova)
 699                         msm_gem_put_iova(a5xx_gpu->pm4_bo, gpu->aspace);
 700                 drm_gem_object_unreference_unlocked(a5xx_gpu->pm4_bo);
 701         }
 702
 703         if (a5xx_gpu->pfp_bo) {
 704                 if (a5xx_gpu->pfp_iova)
 705                         msm_gem_put_iova(a5xx_gpu->pfp_bo, gpu->aspace);
 706                 drm_gem_object_unreference_unlocked(a5xx_gpu->pfp_bo);
 707         }
 708
 709         if (a5xx_gpu->gpmu_bo) {
 710                 if (a5xx_gpu->gpmu_iova)
 711                         msm_gem_put_iova(a5xx_gpu->gpmu_bo, gpu->aspace);
 712                 drm_gem_object_unreference_unlocked(a5xx_gpu->gpmu_bo);
 713         }
 714
 715         adreno_gpu_cleanup(adreno_gpu);
 716         kfree(a5xx_gpu);
 717 }
 718
 719 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
 720 {
 721         if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
 722                 return false;
 723
 724         /*
 725          * Nearly every abnormality ends up pausing the GPU and triggering a
 726          * fault so we can safely just watch for this one interrupt to fire
 727          */
 728         return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
 729                 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
 730 }
 731
 732 bool a5xx_idle(struct msm_gpu *gpu)
 733 {
 734         /* wait for CP to drain ringbuffer: */
 735         if (!adreno_idle(gpu))
 736                 return false;
 737
 738         if (spin_until(_a5xx_check_idle(gpu))) {
 739                 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X\n",
 740                         gpu->name, __builtin_return_address(0),
 741                         gpu_read(gpu, REG_A5XX_RBBM_STATUS),
 742                         gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS));
 743
 744                 return false;
 745         }
 746
 747         return true;
 748 }
 749
 750 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags)
 751 {
 752         struct msm_gpu *gpu = arg;
 753         pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
 754                         iova, flags,
 755                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
 756                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
 757                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
 758                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));
 759
 760         return -EFAULT;
 761 }
 762
 763 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
 764 {
 765         u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
 766
 767         if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
 768                 u32 val;
 769
 770                 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
 771
 772                 /*
 773                  * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
 774                  * read it twice
 775                  */
 776
 777                 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
 778                 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
 779
 780                 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
 781                         val);
 782         }
 783
 784         if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
 785                 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
 786                         gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
 787
 788         if (status & A5XX_CP_INT_CP_DMA_ERROR)
 789                 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
 790
 791         if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
 792                 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
 793
 794                 dev_err_ratelimited(gpu->dev->dev,
 795                         "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
 796                         val & (1 << 24) ? "WRITE" : "READ",
 797                         (val & 0xFFFFF) >> 2, val);
 798         }
 799
 800         if (status & A5XX_CP_INT_CP_AHB_ERROR) {
 801                 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
 802                 const char *access[16] = { "reserved", "reserved",
 803                         "timestamp lo", "timestamp hi", "pfp read", "pfp write",
 804                         "", "", "me read", "me write", "", "", "crashdump read",
 805                         "crashdump write" };
 806
 807                 dev_err_ratelimited(gpu->dev->dev,
 808                         "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
 809                         status & 0xFFFFF, access[(status >> 24) & 0xF],
 810                         (status & (1 << 31)), status);
 811         }
 812 }
 813
 814 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)
 815 {
 816         if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
 817                 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
 818
 819                 dev_err_ratelimited(gpu->dev->dev,
 820                         "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
 821                         val & (1 << 28) ? "WRITE" : "READ",
 822                         (val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
 823                         (val >> 24) & 0xF);
 824
 825                 /* Clear the error */
 826                 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
 827
 828                 /* Clear the interrupt */
 829                 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
 830                         A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
 831         }
 832
 833         if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
 834                 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
 835
 836         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
 837                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
 838                         gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
 839
 840         if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
 841                 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
 842                         gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
 843
 844         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
 845                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
 846                         gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
 847
 848         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
 849                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
 850
 851         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
 852                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
 853 }
 854
 855 static void a5xx_uche_err_irq(struct msm_gpu *gpu)
 856 {
 857         uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
 858
 859         addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
 860
 861         dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
 862                 addr);
 863 }
 864
 865 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
 866 {
 867         dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
 868 }
 869
 870 #define RBBM_ERROR_MASK \
 871         (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
 872         A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
 873         A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
 874         A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
 875         A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
 876         A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
 877
 878 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
 879 {
 880         u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
 881
 882         /*
 883          * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it
 884          * before the source is cleared the interrupt will storm.
 885          */
 886         gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
 887                 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
 888
 889         /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */
 890         if (status & RBBM_ERROR_MASK)
 891                 a5xx_rbbm_err_irq(gpu, status);
 892
 893         if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
 894                 a5xx_cp_err_irq(gpu);
 895
 896         if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
 897                 a5xx_uche_err_irq(gpu);
 898
 899         if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
 900                 a5xx_gpmu_err_irq(gpu);
 901
 902         if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
 903                 msm_gpu_retire(gpu);
 904
 905         return IRQ_HANDLED;
 906 }
 907
 908 static const u32 a5xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
 909         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A5XX_CP_RB_BASE),
 910         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A5XX_CP_RB_BASE_HI),
 911         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A5XX_CP_RB_RPTR_ADDR),
 912         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
 913                 REG_A5XX_CP_RB_RPTR_ADDR_HI),
 914         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A5XX_CP_RB_RPTR),
 915         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A5XX_CP_RB_WPTR),
 916         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A5XX_CP_RB_CNTL),
 917 };
 918
 919 static const u32 a5xx_registers[] = {
 920         0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
 921         0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
 922         0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
 923         0x04E0, 0x0533, 0x0540, 0x0555, 0xF400, 0xF400, 0xF800, 0xF807,
 924         0x0800, 0x081A, 0x081F, 0x0841, 0x0860, 0x0860, 0x0880, 0x08A0,
 925         0x0B00, 0x0B12, 0x0B15, 0x0B28, 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD,
 926         0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 0x0C60, 0x0C61, 0x0C80, 0x0C82,
 927         0x0C84, 0x0C85, 0x0C90, 0x0C98, 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2,
 928         0x2180, 0x2185, 0x2580, 0x2585, 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7,
 929         0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8,
 930         0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, 0x2100, 0x211E, 0x2140, 0x2145,
 931         0x2500, 0x251E, 0x2540, 0x2545, 0x0D10, 0x0D17, 0x0D20, 0x0D23,
 932         0x0D30, 0x0D30, 0x20C0, 0x20C0, 0x24C0, 0x24C0, 0x0E40, 0x0E43,
 933         0x0E4A, 0x0E4A, 0x0E50, 0x0E57, 0x0E60, 0x0E7C, 0x0E80, 0x0E8E,
 934         0x0E90, 0x0E96, 0x0EA0, 0x0EA8, 0x0EB0, 0x0EB2, 0xE140, 0xE147,
 935         0xE150, 0xE187, 0xE1A0, 0xE1A9, 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7,
 936         0xE1D0, 0xE1D1, 0xE200, 0xE201, 0xE210, 0xE21C, 0xE240, 0xE268,
 937         0xE000, 0xE006, 0xE010, 0xE09A, 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB,
 938         0xE100, 0xE105, 0xE380, 0xE38F, 0xE3B0, 0xE3B0, 0xE400, 0xE405,
 939         0xE408, 0xE4E9, 0xE4F0, 0xE4F0, 0xE280, 0xE280, 0xE282, 0xE2A3,
 940         0xE2A5, 0xE2C2, 0xE940, 0xE947, 0xE950, 0xE987, 0xE9A0, 0xE9A9,
 941         0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7, 0xE9D0, 0xE9D1, 0xEA00, 0xEA01,
 942         0xEA10, 0xEA1C, 0xEA40, 0xEA68, 0xE800, 0xE806, 0xE810, 0xE89A,
 943         0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, 0xE900, 0xE905, 0xEB80, 0xEB8F,
 944         0xEBB0, 0xEBB0, 0xEC00, 0xEC05, 0xEC08, 0xECE9, 0xECF0, 0xECF0,
 945         0xEA80, 0xEA80, 0xEA82, 0xEAA3, 0xEAA5, 0xEAC2, 0xA800, 0xA8FF,
 946         0xAC60, 0xAC60, 0xB000, 0xB97F, 0xB9A0, 0xB9BF,
 947         ~0
 948 };
 949
 950 static void a5xx_dump(struct msm_gpu *gpu)
 951 {
 952         dev_info(gpu->dev->dev, "status:   %08x\n",
 953                 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
 954         adreno_dump(gpu);
 955 }
 956
 957 static int a5xx_pm_resume(struct msm_gpu *gpu)
 958 {
 959         int ret;
 960
 961         /* Turn on the core power */
 962         ret = msm_gpu_pm_resume(gpu);
 963         if (ret)
 964                 return ret;
 965
 966         /* Turn the RBCCU domain first to limit the chances of voltage droop */
 967         gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
 968
 969         /* Wait 3 usecs before polling */
 970         udelay(3);
 971
 972         ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
 973                 (1 << 20), (1 << 20));
 974         if (ret) {
 975                 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
 976                         gpu->name,
 977                         gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
 978                 return ret;
 979         }
 980
 981         /* Turn on the SP domain */
 982         gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
 983         ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
 984                 (1 << 20), (1 << 20));
 985         if (ret)
 986                 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
 987                         gpu->name);
 988
 989         return ret;
 990 }
 991
 992 static int a5xx_pm_suspend(struct msm_gpu *gpu)
 993 {
 994         /* Clear the VBIF pipe before shutting down */
 995         gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0xF);
 996         spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 0xF) == 0xF);
 997
 998         gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
 999
1000         /*
1001          * Reset the VBIF before power collapse to avoid issue with FIFO
1002          * entries
1003          */
1004         gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
1005         gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
1006
1007         return msm_gpu_pm_suspend(gpu);
1008 }
1009
1010 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
1011 {
1012         *value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,
1013                 REG_A5XX_RBBM_PERFCTR_CP_0_HI);
1014
1015         return 0;
1016 }
1017
1018 #ifdef CONFIG_DEBUG_FS
1019 static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
1020 {
1021         seq_printf(m, "status:   %08x\n",
1022                         gpu_read(gpu, REG_A5XX_RBBM_STATUS));
1023         adreno_show(gpu, m);
1024 }
1025 #endif
1026
1027 static const struct adreno_gpu_funcs funcs = {
1028         .base = {
1029                 .get_param = adreno_get_param,
1030                 .hw_init = a5xx_hw_init,
1031                 .pm_suspend = a5xx_pm_suspend,
1032                 .pm_resume = a5xx_pm_resume,
1033                 .recover = a5xx_recover,
1034                 .last_fence = adreno_last_fence,
1035                 .submit = a5xx_submit,
1036                 .flush = adreno_flush,
1037                 .irq = a5xx_irq,
1038                 .destroy = a5xx_destroy,
1039 #ifdef CONFIG_DEBUG_FS
1040                 .show = a5xx_show,
1041 #endif
1042         },
1043         .get_timestamp = a5xx_get_timestamp,
1044 };
1045
1046 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
1047 {
1048         struct msm_drm_private *priv = dev->dev_private;
1049         struct platform_device *pdev = priv->gpu_pdev;
1050         struct a5xx_gpu *a5xx_gpu = NULL;
1051         struct adreno_gpu *adreno_gpu;
1052         struct msm_gpu *gpu;
1053         int ret;
1054
1055         if (!pdev) {
1056                 dev_err(dev->dev, "No A5XX device is defined\n");
1057                 return ERR_PTR(-ENXIO);
1058         }
1059
1060         a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
1061         if (!a5xx_gpu)
1062                 return ERR_PTR(-ENOMEM);
1063
1064         adreno_gpu = &a5xx_gpu->base;
1065         gpu = &adreno_gpu->base;
1066
1067         a5xx_gpu->pdev = pdev;
1068         adreno_gpu->registers = a5xx_registers;
1069         adreno_gpu->reg_offsets = a5xx_register_offsets;
1070
1071         a5xx_gpu->lm_leakage = 0x4E001A;
1072
1073         ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs);
1074         if (ret) {
1075                 a5xx_destroy(&(a5xx_gpu->base.base));
1076                 return ERR_PTR(ret);
1077         }
1078
1079         if (gpu->aspace)
1080                 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);
1081
1082         return gpu;
1083 }