drivers/gpu/drm/msm/adreno/a5xx_gpu.c

   1 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License version 2 and
   5  * only version 2 as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  */
  13
  14 #include <linux/types.h>
  15 #include <linux/cpumask.h>
  16 #include <linux/qcom_scm.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/of_address.h>
  19 #include <linux/soc/qcom/mdt_loader.h>
  20 #include "msm_gem.h"
  21 #include "msm_mmu.h"
  22 #include "a5xx_gpu.h"
  23
  24 extern bool hang_debug;
  25 static void a5xx_dump(struct msm_gpu *gpu);
  26
  27 #define GPU_PAS_ID 13
  28
  29 static int zap_shader_load_mdt(struct device *dev, const char *fwname)
  30 {
  31         const struct firmware *fw;
  32         struct device_node *np;
  33         struct resource r;
  34         phys_addr_t mem_phys;
  35         ssize_t mem_size;
  36         void *mem_region = NULL;
  37         int ret;
  38
  39         if (!IS_ENABLED(CONFIG_ARCH_QCOM))
  40                 return -EINVAL;
  41
  42         np = of_get_child_by_name(dev->of_node, "zap-shader");
  43         if (!np)
  44                 return -ENODEV;
  45
  46         np = of_parse_phandle(np, "memory-region", 0);
  47         if (!np)
  48                 return -EINVAL;
  49
  50         ret = of_address_to_resource(np, 0, &r);
  51         if (ret)
  52                 return ret;
  53
  54         mem_phys = r.start;
  55         mem_size = resource_size(&r);
  56
  57         /* Request the MDT file for the firmware */
  58         ret = request_firmware(&fw, fwname, dev);
  59         if (ret) {
  60                 DRM_DEV_ERROR(dev, "Unable to load %s\n", fwname);
  61                 return ret;
  62         }
  63
  64         /* Figure out how much memory we need */
  65         mem_size = qcom_mdt_get_size(fw);
  66         if (mem_size < 0) {
  67                 ret = mem_size;
  68                 goto out;
  69         }
  70
  71         /* Allocate memory for the firmware image */
  72         mem_region = memremap(mem_phys, mem_size,  MEMREMAP_WC);
  73         if (!mem_region) {
  74                 ret = -ENOMEM;
  75                 goto out;
  76         }
  77
  78         /* Load the rest of the MDT */
  79         ret = qcom_mdt_load(dev, fw, fwname, GPU_PAS_ID, mem_region, mem_phys,
  80                 mem_size);
  81         if (ret)
  82                 goto out;
  83
  84         /* Send the image to the secure world */
  85         ret = qcom_scm_pas_auth_and_reset(GPU_PAS_ID);
  86         if (ret)
  87                 DRM_DEV_ERROR(dev, "Unable to authorize the image\n");
  88
  89 out:
  90         if (mem_region)
  91                 memunmap(mem_region);
  92
  93         release_firmware(fw);
  94
  95         return ret;
  96 }
  97
  98 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
  99         struct msm_file_private *ctx)
 100 {
 101         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 102         struct msm_drm_private *priv = gpu->dev->dev_private;
 103         struct msm_ringbuffer *ring = gpu->rb;
 104         unsigned int i, ibs = 0;
 105
 106         for (i = 0; i < submit->nr_cmds; i++) {
 107                 switch (submit->cmd[i].type) {
 108                 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
 109                         break;
 110                 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
 111                         if (priv->lastctx == ctx)
 112                                 break;
 113                 case MSM_SUBMIT_CMD_BUF:
 114                         OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
 115                         OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
 116                         OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
 117                         OUT_RING(ring, submit->cmd[i].size);
 118                         ibs++;
 119                         break;
 120                 }
 121         }
 122
 123         OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
 124         OUT_RING(ring, submit->fence->seqno);
 125
 126         OUT_PKT7(ring, CP_EVENT_WRITE, 4);
 127         OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
 128         OUT_RING(ring, lower_32_bits(rbmemptr(adreno_gpu, fence)));
 129         OUT_RING(ring, upper_32_bits(rbmemptr(adreno_gpu, fence)));
 130         OUT_RING(ring, submit->fence->seqno);
 131
 132         gpu->funcs->flush(gpu);
 133 }
 134
 135 static const struct {
 136         u32 offset;
 137         u32 value;
 138 } a5xx_hwcg[] = {
 139         {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 140         {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
 141         {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
 142         {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
 143         {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 144         {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
 145         {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
 146         {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
 147         {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 148         {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
 149         {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
 150         {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
 151         {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 152         {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
 153         {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
 154         {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
 155         {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
 156         {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
 157         {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
 158         {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
 159         {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 160         {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
 161         {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
 162         {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
 163         {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
 164         {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
 165         {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
 166         {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
 167         {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 168         {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
 169         {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
 170         {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
 171         {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 172         {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
 173         {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
 174         {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
 175         {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
 176         {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
 177         {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
 178         {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
 179         {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 180         {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
 181         {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
 182         {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
 183         {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 184         {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
 185         {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
 186         {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
 187         {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
 188         {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
 189         {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
 190         {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
 191         {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 192         {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
 193         {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
 194         {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
 195         {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
 196         {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 197         {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 198         {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
 199         {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
 200         {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
 201         {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
 202         {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
 203         {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
 204         {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
 205         {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
 206         {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
 207         {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
 208         {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
 209         {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
 210         {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
 211         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
 212         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
 213         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
 214         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
 215         {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
 216         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
 217         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
 218         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
 219         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
 220         {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
 221         {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 222         {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
 223         {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 224         {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 225         {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 226         {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 227         {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 228         {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 229         {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 230         {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
 231 };
 232
 233 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)
 234 {
 235         unsigned int i;
 236
 237         for (i = 0; i < ARRAY_SIZE(a5xx_hwcg); i++)
 238                 gpu_write(gpu, a5xx_hwcg[i].offset,
 239                         state ? a5xx_hwcg[i].value : 0);
 240
 241         gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);
 242         gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);
 243 }
 244
 245 static int a5xx_me_init(struct msm_gpu *gpu)
 246 {
 247         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 248         struct msm_ringbuffer *ring = gpu->rb;
 249
 250         OUT_PKT7(ring, CP_ME_INIT, 8);
 251
 252         OUT_RING(ring, 0x0000002F);
 253
 254         /* Enable multiple hardware contexts */
 255         OUT_RING(ring, 0x00000003);
 256
 257         /* Enable error detection */
 258         OUT_RING(ring, 0x20000000);
 259
 260         /* Don't enable header dump */
 261         OUT_RING(ring, 0x00000000);
 262         OUT_RING(ring, 0x00000000);
 263
 264         /* Specify workarounds for various microcode issues */
 265         if (adreno_is_a530(adreno_gpu)) {
 266                 /* Workaround for token end syncs
 267                  * Force a WFI after every direct-render 3D mode draw and every
 268                  * 2D mode 3 draw
 269                  */
 270                 OUT_RING(ring, 0x0000000B);
 271         } else {
 272                 /* No workarounds enabled */
 273                 OUT_RING(ring, 0x00000000);
 274         }
 275
 276         OUT_RING(ring, 0x00000000);
 277         OUT_RING(ring, 0x00000000);
 278
 279         gpu->funcs->flush(gpu);
 280
 281         return a5xx_idle(gpu) ? 0 : -EINVAL;
 282 }
 283
 284 static struct drm_gem_object *a5xx_ucode_load_bo(struct msm_gpu *gpu,
 285                 const struct firmware *fw, u64 *iova)
 286 {
 287         struct drm_device *drm = gpu->dev;
 288         struct drm_gem_object *bo;
 289         void *ptr;
 290
 291         bo = msm_gem_new_locked(drm, fw->size - 4, MSM_BO_UNCACHED);
 292         if (IS_ERR(bo))
 293                 return bo;
 294
 295         ptr = msm_gem_get_vaddr(bo);
 296         if (!ptr) {
 297                 drm_gem_object_unreference(bo);
 298                 return ERR_PTR(-ENOMEM);
 299         }
 300
 301         if (iova) {
 302                 int ret = msm_gem_get_iova(bo, gpu->aspace, iova);
 303
 304                 if (ret) {
 305                         drm_gem_object_unreference(bo);
 306                         return ERR_PTR(ret);
 307                 }
 308         }
 309
 310         memcpy(ptr, &fw->data[4], fw->size - 4);
 311
 312         msm_gem_put_vaddr(bo);
 313         return bo;
 314 }
 315
 316 static int a5xx_ucode_init(struct msm_gpu *gpu)
 317 {
 318         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 319         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 320         int ret;
 321
 322         if (!a5xx_gpu->pm4_bo) {
 323                 a5xx_gpu->pm4_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pm4,
 324                         &a5xx_gpu->pm4_iova);
 325
 326                 if (IS_ERR(a5xx_gpu->pm4_bo)) {
 327                         ret = PTR_ERR(a5xx_gpu->pm4_bo);
 328                         a5xx_gpu->pm4_bo = NULL;
 329                         dev_err(gpu->dev->dev, "could not allocate PM4: %d\n",
 330                                 ret);
 331                         return ret;
 332                 }
 333         }
 334
 335         if (!a5xx_gpu->pfp_bo) {
 336                 a5xx_gpu->pfp_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pfp,
 337                         &a5xx_gpu->pfp_iova);
 338
 339                 if (IS_ERR(a5xx_gpu->pfp_bo)) {
 340                         ret = PTR_ERR(a5xx_gpu->pfp_bo);
 341                         a5xx_gpu->pfp_bo = NULL;
 342                         dev_err(gpu->dev->dev, "could not allocate PFP: %d\n",
 343                                 ret);
 344                         return ret;
 345                 }
 346         }
 347
 348         gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,
 349                 REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova);
 350
 351         gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,
 352                 REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova);
 353
 354         return 0;
 355 }
 356
 357 #define SCM_GPU_ZAP_SHADER_RESUME 0
 358
 359 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
 360 {
 361         int ret;
 362
 363         ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
 364         if (ret)
 365                 DRM_ERROR("%s: zap-shader resume failed: %d\n",
 366                         gpu->name, ret);
 367
 368         return ret;
 369 }
 370
 371 static int a5xx_zap_shader_init(struct msm_gpu *gpu)
 372 {
 373         static bool loaded;
 374         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 375         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 376         struct platform_device *pdev = a5xx_gpu->pdev;
 377         int ret;
 378
 379         /*
 380          * If the zap shader is already loaded into memory we just need to kick
 381          * the remote processor to reinitialize it
 382          */
 383         if (loaded)
 384                 return a5xx_zap_shader_resume(gpu);
 385
 386         /* We need SCM to be able to load the firmware */
 387         if (!qcom_scm_is_available()) {
 388                 DRM_DEV_ERROR(&pdev->dev, "SCM is not available\n");
 389                 return -EPROBE_DEFER;
 390         }
 391
 392         /* Each GPU has a target specific zap shader firmware name to use */
 393         if (!adreno_gpu->info->zapfw) {
 394                 DRM_DEV_ERROR(&pdev->dev,
 395                         "Zap shader firmware file not specified for this target\n");
 396                 return -ENODEV;
 397         }
 398
 399         ret = zap_shader_load_mdt(&pdev->dev, adreno_gpu->info->zapfw);
 400
 401         loaded = !ret;
 402
 403         return ret;
 404 }
 405
 406 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
 407           A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
 408           A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
 409           A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
 410           A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
 411           A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
 412           A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
 413           A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
 414           A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
 415           A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
 416
 417 static int a5xx_hw_init(struct msm_gpu *gpu)
 418 {
 419         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 420         int ret;
 421
 422         gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
 423
 424         /* Make all blocks contribute to the GPU BUSY perf counter */
 425         gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
 426
 427         /* Enable RBBM error reporting bits */
 428         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
 429
 430         if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
 431                 /*
 432                  * Mask out the activity signals from RB1-3 to avoid false
 433                  * positives
 434                  */
 435
 436                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
 437                         0xF0000000);
 438                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
 439                         0xFFFFFFFF);
 440                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
 441                         0xFFFFFFFF);
 442                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
 443                         0xFFFFFFFF);
 444                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
 445                         0xFFFFFFFF);
 446                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
 447                         0xFFFFFFFF);
 448                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
 449                         0xFFFFFFFF);
 450                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
 451                         0xFFFFFFFF);
 452         }
 453
 454         /* Enable fault detection */
 455         gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
 456                 (1 << 30) | 0xFFFF);
 457
 458         /* Turn on performance counters */
 459         gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
 460
 461         /* Increase VFD cache access so LRZ and other data gets evicted less */
 462         gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
 463
 464         /* Disable L2 bypass in the UCHE */
 465         gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
 466         gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
 467         gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
 468         gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
 469
 470         /* Set the GMEM VA range (0 to gpu->gmem) */
 471         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
 472         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
 473         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
 474                 0x00100000 + adreno_gpu->gmem - 1);
 475         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
 476
 477         gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
 478         gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
 479         gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
 480         gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
 481
 482         gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22));
 483
 484         if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
 485                 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
 486
 487         gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100);
 488
 489         /* Enable USE_RETENTION_FLOPS */
 490         gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
 491
 492         /* Enable ME/PFP split notification */
 493         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
 494
 495         /* Enable HWCG */
 496         a5xx_set_hwcg(gpu, true);
 497
 498         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
 499
 500         /* Set the highest bank bit */
 501         gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7);
 502         gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1);
 503
 504         /* Protect registers from the CP */
 505         gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
 506
 507         /* RBBM */
 508         gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
 509         gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
 510         gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
 511         gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
 512         gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
 513         gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
 514
 515         /* Content protect */
 516         gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
 517                 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
 518                         16));
 519         gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
 520                 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
 521
 522         /* CP */
 523         gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
 524         gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
 525         gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
 526         gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
 527
 528         /* RB */
 529         gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
 530         gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
 531
 532         /* VPC */
 533         gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
 534         gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4));
 535
 536         /* UCHE */
 537         gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
 538
 539         if (adreno_is_a530(adreno_gpu))
 540                 gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
 541                         ADRENO_PROTECT_RW(0x10000, 0x8000));
 542
 543         gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
 544         /*
 545          * Disable the trusted memory range - we don't actually supported secure
 546          * memory rendering at this point in time and we don't want to block off
 547          * part of the virtual memory space.
 548          */
 549         gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
 550                 REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
 551         gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
 552
 553         /* Load the GPMU firmware before starting the HW init */
 554         a5xx_gpmu_ucode_init(gpu);
 555
 556         ret = adreno_hw_init(gpu);
 557         if (ret)
 558                 return ret;
 559
 560         ret = a5xx_ucode_init(gpu);
 561         if (ret)
 562                 return ret;
 563
 564         /* Disable the interrupts through the initial bringup stage */
 565         gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
 566
 567         /* Clear ME_HALT to start the micro engine */
 568         gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
 569         ret = a5xx_me_init(gpu);
 570         if (ret)
 571                 return ret;
 572
 573         ret = a5xx_power_init(gpu);
 574         if (ret)
 575                 return ret;
 576
 577         /*
 578          * Send a pipeline event stat to get misbehaving counters to start
 579          * ticking correctly
 580          */
 581         if (adreno_is_a530(adreno_gpu)) {
 582                 OUT_PKT7(gpu->rb, CP_EVENT_WRITE, 1);
 583                 OUT_RING(gpu->rb, 0x0F);
 584
 585                 gpu->funcs->flush(gpu);
 586                 if (!a5xx_idle(gpu))
 587                         return -EINVAL;
 588         }
 589
 590         /*
 591          * Try to load a zap shader into the secure world. If successful
 592          * we can use the CP to switch out of secure mode. If not then we
 593          * have no resource but to try to switch ourselves out manually. If we
 594          * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
 595          * be blocked and a permissions violation will soon follow.
 596          */
 597         ret = a5xx_zap_shader_init(gpu);
 598         if (!ret) {
 599                 OUT_PKT7(gpu->rb, CP_SET_SECURE_MODE, 1);
 600                 OUT_RING(gpu->rb, 0x00000000);
 601
 602                 gpu->funcs->flush(gpu);
 603                 if (!a5xx_idle(gpu))
 604                         return -EINVAL;
 605         } else {
 606                 /* Print a warning so if we die, we know why */
 607                 dev_warn_once(gpu->dev->dev,
 608                         "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
 609                 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
 610         }
 611
 612         return 0;
 613 }
 614
 615 static void a5xx_recover(struct msm_gpu *gpu)
 616 {
 617         int i;
 618
 619         adreno_dump_info(gpu);
 620
 621         for (i = 0; i < 8; i++) {
 622                 printk("CP_SCRATCH_REG%d: %u\n", i,
 623                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
 624         }
 625
 626         if (hang_debug)
 627                 a5xx_dump(gpu);
 628
 629         gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
 630         gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
 631         gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
 632         adreno_recover(gpu);
 633 }
 634
 635 static void a5xx_destroy(struct msm_gpu *gpu)
 636 {
 637         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 638         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 639
 640         DBG("%s", gpu->name);
 641
 642         if (a5xx_gpu->pm4_bo) {
 643                 if (a5xx_gpu->pm4_iova)
 644                         msm_gem_put_iova(a5xx_gpu->pm4_bo, gpu->aspace);
 645                 drm_gem_object_unreference_unlocked(a5xx_gpu->pm4_bo);
 646         }
 647
 648         if (a5xx_gpu->pfp_bo) {
 649                 if (a5xx_gpu->pfp_iova)
 650                         msm_gem_put_iova(a5xx_gpu->pfp_bo, gpu->aspace);
 651                 drm_gem_object_unreference_unlocked(a5xx_gpu->pfp_bo);
 652         }
 653
 654         if (a5xx_gpu->gpmu_bo) {
 655                 if (a5xx_gpu->gpmu_iova)
 656                         msm_gem_put_iova(a5xx_gpu->gpmu_bo, gpu->aspace);
 657                 drm_gem_object_unreference_unlocked(a5xx_gpu->gpmu_bo);
 658         }
 659
 660         adreno_gpu_cleanup(adreno_gpu);
 661         kfree(a5xx_gpu);
 662 }
 663
 664 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
 665 {
 666         if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
 667                 return false;
 668
 669         /*
 670          * Nearly every abnormality ends up pausing the GPU and triggering a
 671          * fault so we can safely just watch for this one interrupt to fire
 672          */
 673         return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
 674                 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
 675 }
 676
 677 bool a5xx_idle(struct msm_gpu *gpu)
 678 {
 679         /* wait for CP to drain ringbuffer: */
 680         if (!adreno_idle(gpu))
 681                 return false;
 682
 683         if (spin_until(_a5xx_check_idle(gpu))) {
 684                 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X\n",
 685                         gpu->name, __builtin_return_address(0),
 686                         gpu_read(gpu, REG_A5XX_RBBM_STATUS),
 687                         gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS));
 688
 689                 return false;
 690         }
 691
 692         return true;
 693 }
 694
 695 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags)
 696 {
 697         struct msm_gpu *gpu = arg;
 698         pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
 699                         iova, flags,
 700                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
 701                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
 702                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
 703                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));
 704
 705         return -EFAULT;
 706 }
 707
 708 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
 709 {
 710         u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
 711
 712         if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
 713                 u32 val;
 714
 715                 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
 716
 717                 /*
 718                  * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
 719                  * read it twice
 720                  */
 721
 722                 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
 723                 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
 724
 725                 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
 726                         val);
 727         }
 728
 729         if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
 730                 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
 731                         gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
 732
 733         if (status & A5XX_CP_INT_CP_DMA_ERROR)
 734                 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
 735
 736         if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
 737                 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
 738
 739                 dev_err_ratelimited(gpu->dev->dev,
 740                         "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
 741                         val & (1 << 24) ? "WRITE" : "READ",
 742                         (val & 0xFFFFF) >> 2, val);
 743         }
 744
 745         if (status & A5XX_CP_INT_CP_AHB_ERROR) {
 746                 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
 747                 const char *access[16] = { "reserved", "reserved",
 748                         "timestamp lo", "timestamp hi", "pfp read", "pfp write",
 749                         "", "", "me read", "me write", "", "", "crashdump read",
 750                         "crashdump write" };
 751
 752                 dev_err_ratelimited(gpu->dev->dev,
 753                         "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
 754                         status & 0xFFFFF, access[(status >> 24) & 0xF],
 755                         (status & (1 << 31)), status);
 756         }
 757 }
 758
 759 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)
 760 {
 761         if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
 762                 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
 763
 764                 dev_err_ratelimited(gpu->dev->dev,
 765                         "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
 766                         val & (1 << 28) ? "WRITE" : "READ",
 767                         (val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
 768                         (val >> 24) & 0xF);
 769
 770                 /* Clear the error */
 771                 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
 772
 773                 /* Clear the interrupt */
 774                 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
 775                         A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
 776         }
 777
 778         if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
 779                 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
 780
 781         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
 782                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
 783                         gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
 784
 785         if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
 786                 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
 787                         gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
 788
 789         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
 790                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
 791                         gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
 792
 793         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
 794                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
 795
 796         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
 797                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
 798 }
 799
 800 static void a5xx_uche_err_irq(struct msm_gpu *gpu)
 801 {
 802         uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
 803
 804         addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
 805
 806         dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
 807                 addr);
 808 }
 809
 810 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
 811 {
 812         dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
 813 }
 814
 815 #define RBBM_ERROR_MASK \
 816         (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
 817         A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
 818         A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
 819         A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
 820         A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
 821         A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
 822
 823 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
 824 {
 825         u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
 826
 827         /*
 828          * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it
 829          * before the source is cleared the interrupt will storm.
 830          */
 831         gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
 832                 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
 833
 834         /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */
 835         if (status & RBBM_ERROR_MASK)
 836                 a5xx_rbbm_err_irq(gpu, status);
 837
 838         if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
 839                 a5xx_cp_err_irq(gpu);
 840
 841         if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
 842                 a5xx_uche_err_irq(gpu);
 843
 844         if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
 845                 a5xx_gpmu_err_irq(gpu);
 846
 847         if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
 848                 msm_gpu_retire(gpu);
 849
 850         return IRQ_HANDLED;
 851 }
 852
 853 static const u32 a5xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
 854         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A5XX_CP_RB_BASE),
 855         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A5XX_CP_RB_BASE_HI),
 856         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A5XX_CP_RB_RPTR_ADDR),
 857         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
 858                 REG_A5XX_CP_RB_RPTR_ADDR_HI),
 859         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A5XX_CP_RB_RPTR),
 860         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A5XX_CP_RB_WPTR),
 861         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A5XX_CP_RB_CNTL),
 862 };
 863
 864 static const u32 a5xx_registers[] = {
 865         0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
 866         0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
 867         0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
 868         0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841,
 869         0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28,
 870         0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53,
 871         0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98,
 872         0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585,
 873         0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8,
 874         0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E,
 875         0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545,
 876         0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0,
 877         0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57,
 878         0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8,
 879         0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9,
 880         0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201,
 881         0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A,
 882         0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F,
 883         0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0,
 884         0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947,
 885         0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7,
 886         0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68,
 887         0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB,
 888         0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05,
 889         0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3,
 890         0xEAA5, 0xEAC2, 0xA800, 0xA8FF, 0xAC60, 0xAC60, 0xB000, 0xB97F,
 891         0xB9A0, 0xB9BF, ~0
 892 };
 893
 894 static void a5xx_dump(struct msm_gpu *gpu)
 895 {
 896         dev_info(gpu->dev->dev, "status:   %08x\n",
 897                 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
 898         adreno_dump(gpu);
 899 }
 900
 901 static int a5xx_pm_resume(struct msm_gpu *gpu)
 902 {
 903         int ret;
 904
 905         /* Turn on the core power */
 906         ret = msm_gpu_pm_resume(gpu);
 907         if (ret)
 908                 return ret;
 909
 910         /* Turn the RBCCU domain first to limit the chances of voltage droop */
 911         gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
 912
 913         /* Wait 3 usecs before polling */
 914         udelay(3);
 915
 916         ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
 917                 (1 << 20), (1 << 20));
 918         if (ret) {
 919                 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
 920                         gpu->name,
 921                         gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
 922                 return ret;
 923         }
 924
 925         /* Turn on the SP domain */
 926         gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
 927         ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
 928                 (1 << 20), (1 << 20));
 929         if (ret)
 930                 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
 931                         gpu->name);
 932
 933         return ret;
 934 }
 935
 936 static int a5xx_pm_suspend(struct msm_gpu *gpu)
 937 {
 938         /* Clear the VBIF pipe before shutting down */
 939         gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0xF);
 940         spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 0xF) == 0xF);
 941
 942         gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
 943
 944         /*
 945          * Reset the VBIF before power collapse to avoid issue with FIFO
 946          * entries
 947          */
 948         gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
 949         gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
 950
 951         return msm_gpu_pm_suspend(gpu);
 952 }
 953
 954 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 955 {
 956         *value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,
 957                 REG_A5XX_RBBM_PERFCTR_CP_0_HI);
 958
 959         return 0;
 960 }
 961
 962 #ifdef CONFIG_DEBUG_FS
 963 static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
 964 {
 965         seq_printf(m, "status:   %08x\n",
 966                         gpu_read(gpu, REG_A5XX_RBBM_STATUS));
 967
 968         /*
 969          * Temporarily disable hardware clock gating before going into
 970          * adreno_show to avoid issues while reading the registers
 971          */
 972         a5xx_set_hwcg(gpu, false);
 973         adreno_show(gpu, m);
 974         a5xx_set_hwcg(gpu, true);
 975 }
 976 #endif
 977
 978 static const struct adreno_gpu_funcs funcs = {
 979         .base = {
 980                 .get_param = adreno_get_param,
 981                 .hw_init = a5xx_hw_init,
 982                 .pm_suspend = a5xx_pm_suspend,
 983                 .pm_resume = a5xx_pm_resume,
 984                 .recover = a5xx_recover,
 985                 .last_fence = adreno_last_fence,
 986                 .submit = a5xx_submit,
 987                 .flush = adreno_flush,
 988                 .irq = a5xx_irq,
 989                 .destroy = a5xx_destroy,
 990 #ifdef CONFIG_DEBUG_FS
 991                 .show = a5xx_show,
 992 #endif
 993         },
 994         .get_timestamp = a5xx_get_timestamp,
 995 };
 996
 997 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
 998 {
 999         struct msm_drm_private *priv = dev->dev_private;
1000         struct platform_device *pdev = priv->gpu_pdev;
1001         struct a5xx_gpu *a5xx_gpu = NULL;
1002         struct adreno_gpu *adreno_gpu;
1003         struct msm_gpu *gpu;
1004         int ret;
1005
1006         if (!pdev) {
1007                 dev_err(dev->dev, "No A5XX device is defined\n");
1008                 return ERR_PTR(-ENXIO);
1009         }
1010
1011         a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
1012         if (!a5xx_gpu)
1013                 return ERR_PTR(-ENOMEM);
1014
1015         adreno_gpu = &a5xx_gpu->base;
1016         gpu = &adreno_gpu->base;
1017
1018         a5xx_gpu->pdev = pdev;
1019         adreno_gpu->registers = a5xx_registers;
1020         adreno_gpu->reg_offsets = a5xx_register_offsets;
1021
1022         a5xx_gpu->lm_leakage = 0x4E001A;
1023
1024         ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs);
1025         if (ret) {
1026                 a5xx_destroy(&(a5xx_gpu->base.base));
1027                 return ERR_PTR(ret);
1028         }
1029
1030         if (gpu->aspace)
1031                 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);
1032
1033         return gpu;
1034 }