drivers/gpu/drm/msm/adreno/a5xx_gpu.c

   1 /* Copyright (c) 2016 The Linux Foundation. All rights reserved.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License version 2 and
   5  * only version 2 as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  */
  13
  14 #include "msm_gem.h"
  15 #include "a5xx_gpu.h"
  16
  17 extern bool hang_debug;
  18 static void a5xx_dump(struct msm_gpu *gpu);
  19
  20 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
  21         struct msm_file_private *ctx)
  22 {
  23         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
  24         struct msm_drm_private *priv = gpu->dev->dev_private;
  25         struct msm_ringbuffer *ring = gpu->rb;
  26         unsigned int i, ibs = 0;
  27
  28         for (i = 0; i < submit->nr_cmds; i++) {
  29                 switch (submit->cmd[i].type) {
  30                 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
  31                         break;
  32                 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
  33                         if (priv->lastctx == ctx)
  34                                 break;
  35                 case MSM_SUBMIT_CMD_BUF:
  36                         OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
  37                         OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
  38                         OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
  39                         OUT_RING(ring, submit->cmd[i].size);
  40                         ibs++;
  41                         break;
  42                 }
  43         }
  44
  45         OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
  46         OUT_RING(ring, submit->fence->seqno);
  47
  48         OUT_PKT7(ring, CP_EVENT_WRITE, 4);
  49         OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
  50         OUT_RING(ring, lower_32_bits(rbmemptr(adreno_gpu, fence)));
  51         OUT_RING(ring, upper_32_bits(rbmemptr(adreno_gpu, fence)));
  52         OUT_RING(ring, submit->fence->seqno);
  53
  54         gpu->funcs->flush(gpu);
  55 }
  56
  57 struct a5xx_hwcg {
  58         u32 offset;
  59         u32 value;
  60 };
  61
  62 static const struct a5xx_hwcg a530_hwcg[] = {
  63         {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
  64         {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
  65         {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
  66         {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
  67         {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
  68         {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
  69         {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
  70         {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
  71         {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
  72         {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
  73         {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
  74         {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
  75         {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
  76         {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
  77         {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
  78         {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
  79         {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
  80         {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
  81         {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
  82         {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
  83         {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
  84         {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
  85         {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
  86         {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
  87         {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
  88         {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
  89         {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
  90         {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
  91         {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
  92         {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
  93         {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
  94         {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
  95         {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
  96         {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
  97         {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
  98         {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
  99         {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
 100         {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
 101         {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
 102         {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
 103         {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 104         {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
 105         {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
 106         {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
 107         {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 108         {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
 109         {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
 110         {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
 111         {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
 112         {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
 113         {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
 114         {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
 115         {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 116         {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
 117         {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
 118         {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
 119         {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
 120         {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 121         {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 122         {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
 123         {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
 124         {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
 125         {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
 126         {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
 127         {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
 128         {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
 129         {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
 130         {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
 131         {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
 132         {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
 133         {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
 134         {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
 135         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
 136         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
 137         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
 138         {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
 139         {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
 140         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
 141         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
 142         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
 143         {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
 144         {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
 145         {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 146         {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
 147         {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 148         {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 149         {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 150         {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 151         {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 152         {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 153         {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 154         {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
 155 };
 156
 157 static const struct {
 158         int (*test)(struct adreno_gpu *gpu);
 159         const struct a5xx_hwcg *regs;
 160         unsigned int count;
 161 } a5xx_hwcg_regs[] = {
 162         { adreno_is_a530, a530_hwcg, ARRAY_SIZE(a530_hwcg), },
 163 };
 164
 165 static void _a5xx_enable_hwcg(struct msm_gpu *gpu,
 166                 const struct a5xx_hwcg *regs, unsigned int count)
 167 {
 168         unsigned int i;
 169
 170         for (i = 0; i < count; i++)
 171                 gpu_write(gpu, regs[i].offset, regs[i].value);
 172
 173         gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xAAA8AA00);
 174         gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, 0x182);
 175 }
 176
 177 static void a5xx_enable_hwcg(struct msm_gpu *gpu)
 178 {
 179         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 180         unsigned int i;
 181
 182         for (i = 0; i < ARRAY_SIZE(a5xx_hwcg_regs); i++) {
 183                 if (a5xx_hwcg_regs[i].test(adreno_gpu)) {
 184                         _a5xx_enable_hwcg(gpu, a5xx_hwcg_regs[i].regs,
 185                                 a5xx_hwcg_regs[i].count);
 186                         return;
 187                 }
 188         }
 189 }
 190
 191 static int a5xx_me_init(struct msm_gpu *gpu)
 192 {
 193         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 194         struct msm_ringbuffer *ring = gpu->rb;
 195
 196         OUT_PKT7(ring, CP_ME_INIT, 8);
 197
 198         OUT_RING(ring, 0x0000002F);
 199
 200         /* Enable multiple hardware contexts */
 201         OUT_RING(ring, 0x00000003);
 202
 203         /* Enable error detection */
 204         OUT_RING(ring, 0x20000000);
 205
 206         /* Don't enable header dump */
 207         OUT_RING(ring, 0x00000000);
 208         OUT_RING(ring, 0x00000000);
 209
 210         /* Specify workarounds for various microcode issues */
 211         if (adreno_is_a530(adreno_gpu)) {
 212                 /* Workaround for token end syncs
 213                  * Force a WFI after every direct-render 3D mode draw and every
 214                  * 2D mode 3 draw
 215                  */
 216                 OUT_RING(ring, 0x0000000B);
 217         } else {
 218                 /* No workarounds enabled */
 219                 OUT_RING(ring, 0x00000000);
 220         }
 221
 222         OUT_RING(ring, 0x00000000);
 223         OUT_RING(ring, 0x00000000);
 224
 225         gpu->funcs->flush(gpu);
 226
 227         return gpu->funcs->idle(gpu) ? 0 : -EINVAL;
 228 }
 229
 230 static struct drm_gem_object *a5xx_ucode_load_bo(struct msm_gpu *gpu,
 231                 const struct firmware *fw, u64 *iova)
 232 {
 233         struct drm_device *drm = gpu->dev;
 234         struct drm_gem_object *bo;
 235         void *ptr;
 236
 237         mutex_lock(&drm->struct_mutex);
 238         bo = msm_gem_new(drm, fw->size - 4, MSM_BO_UNCACHED);
 239         mutex_unlock(&drm->struct_mutex);
 240
 241         if (IS_ERR(bo))
 242                 return bo;
 243
 244         ptr = msm_gem_get_vaddr(bo);
 245         if (!ptr) {
 246                 drm_gem_object_unreference_unlocked(bo);
 247                 return ERR_PTR(-ENOMEM);
 248         }
 249
 250         if (iova) {
 251                 int ret = msm_gem_get_iova(bo, gpu->id, iova);
 252
 253                 if (ret) {
 254                         drm_gem_object_unreference_unlocked(bo);
 255                         return ERR_PTR(ret);
 256                 }
 257         }
 258
 259         memcpy(ptr, &fw->data[4], fw->size - 4);
 260
 261         msm_gem_put_vaddr(bo);
 262         return bo;
 263 }
 264
 265 static int a5xx_ucode_init(struct msm_gpu *gpu)
 266 {
 267         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 268         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 269         int ret;
 270
 271         if (!a5xx_gpu->pm4_bo) {
 272                 a5xx_gpu->pm4_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pm4,
 273                         &a5xx_gpu->pm4_iova);
 274
 275                 if (IS_ERR(a5xx_gpu->pm4_bo)) {
 276                         ret = PTR_ERR(a5xx_gpu->pm4_bo);
 277                         a5xx_gpu->pm4_bo = NULL;
 278                         dev_err(gpu->dev->dev, "could not allocate PM4: %d\n",
 279                                 ret);
 280                         return ret;
 281                 }
 282         }
 283
 284         if (!a5xx_gpu->pfp_bo) {
 285                 a5xx_gpu->pfp_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pfp,
 286                         &a5xx_gpu->pfp_iova);
 287
 288                 if (IS_ERR(a5xx_gpu->pfp_bo)) {
 289                         ret = PTR_ERR(a5xx_gpu->pfp_bo);
 290                         a5xx_gpu->pfp_bo = NULL;
 291                         dev_err(gpu->dev->dev, "could not allocate PFP: %d\n",
 292                                 ret);
 293                         return ret;
 294                 }
 295         }
 296
 297         gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,
 298                 REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova);
 299
 300         gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,
 301                 REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova);
 302
 303         return 0;
 304 }
 305
 306 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
 307           A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
 308           A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
 309           A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
 310           A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
 311           A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
 312           A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
 313           A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
 314           A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
 315           A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
 316
 317 static int a5xx_hw_init(struct msm_gpu *gpu)
 318 {
 319         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 320         int ret;
 321
 322         gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
 323
 324         /* Make all blocks contribute to the GPU BUSY perf counter */
 325         gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
 326
 327         /* Enable RBBM error reporting bits */
 328         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
 329
 330         if (adreno_gpu->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
 331                 /*
 332                  * Mask out the activity signals from RB1-3 to avoid false
 333                  * positives
 334                  */
 335
 336                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
 337                         0xF0000000);
 338                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
 339                         0xFFFFFFFF);
 340                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
 341                         0xFFFFFFFF);
 342                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
 343                         0xFFFFFFFF);
 344                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
 345                         0xFFFFFFFF);
 346                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
 347                         0xFFFFFFFF);
 348                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
 349                         0xFFFFFFFF);
 350                 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
 351                         0xFFFFFFFF);
 352         }
 353
 354         /* Enable fault detection */
 355         gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
 356                 (1 << 30) | 0xFFFF);
 357
 358         /* Turn on performance counters */
 359         gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
 360
 361         /* Increase VFD cache access so LRZ and other data gets evicted less */
 362         gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
 363
 364         /* Disable L2 bypass in the UCHE */
 365         gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
 366         gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
 367         gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
 368         gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
 369
 370         /* Set the GMEM VA range (0 to gpu->gmem) */
 371         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
 372         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
 373         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
 374                 0x00100000 + adreno_gpu->gmem - 1);
 375         gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
 376
 377         gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
 378         gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
 379         gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
 380         gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
 381
 382         gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22));
 383
 384         if (adreno_gpu->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
 385                 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
 386
 387         gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100);
 388
 389         /* Enable USE_RETENTION_FLOPS */
 390         gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
 391
 392         /* Enable ME/PFP split notification */
 393         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
 394
 395         /* Enable HWCG */
 396         a5xx_enable_hwcg(gpu);
 397
 398         gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
 399
 400         /* Set the highest bank bit */
 401         gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7);
 402         gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1);
 403
 404         /* Protect registers from the CP */
 405         gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
 406
 407         /* RBBM */
 408         gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
 409         gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
 410         gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
 411         gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
 412         gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
 413         gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
 414
 415         /* Content protect */
 416         gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
 417                 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
 418                         16));
 419         gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
 420                 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
 421
 422         /* CP */
 423         gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
 424         gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
 425         gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
 426         gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
 427
 428         /* RB */
 429         gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
 430         gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
 431
 432         /* VPC */
 433         gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
 434         gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4));
 435
 436         /* UCHE */
 437         gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
 438
 439         if (adreno_is_a530(adreno_gpu))
 440                 gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
 441                         ADRENO_PROTECT_RW(0x10000, 0x8000));
 442
 443         gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
 444         /*
 445          * Disable the trusted memory range - we don't actually supported secure
 446          * memory rendering at this point in time and we don't want to block off
 447          * part of the virtual memory space.
 448          */
 449         gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
 450                 REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
 451         gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
 452
 453         ret = adreno_hw_init(gpu);
 454         if (ret)
 455                 return ret;
 456
 457         ret = a5xx_ucode_init(gpu);
 458         if (ret)
 459                 return ret;
 460
 461         /* Disable the interrupts through the initial bringup stage */
 462         gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
 463
 464         /* Clear ME_HALT to start the micro engine */
 465         gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
 466         ret = a5xx_me_init(gpu);
 467         if (ret)
 468                 return ret;
 469
 470         /* Put the GPU into insecure mode */
 471         gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
 472
 473         /*
 474          * Send a pipeline event stat to get misbehaving counters to start
 475          * ticking correctly
 476          */
 477         if (adreno_is_a530(adreno_gpu)) {
 478                 OUT_PKT7(gpu->rb, CP_EVENT_WRITE, 1);
 479                 OUT_RING(gpu->rb, 0x0F);
 480
 481                 gpu->funcs->flush(gpu);
 482                 if (!gpu->funcs->idle(gpu))
 483                         return -EINVAL;
 484         }
 485
 486         return 0;
 487 }
 488
 489 static void a5xx_recover(struct msm_gpu *gpu)
 490 {
 491         int i;
 492
 493         adreno_dump_info(gpu);
 494
 495         for (i = 0; i < 8; i++) {
 496                 printk("CP_SCRATCH_REG%d: %u\n", i,
 497                         gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
 498         }
 499
 500         if (hang_debug)
 501                 a5xx_dump(gpu);
 502
 503         gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
 504         gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
 505         gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
 506         adreno_recover(gpu);
 507 }
 508
 509 static void a5xx_destroy(struct msm_gpu *gpu)
 510 {
 511         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 512         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 513
 514         DBG("%s", gpu->name);
 515
 516         if (a5xx_gpu->pm4_bo) {
 517                 if (a5xx_gpu->pm4_iova)
 518                         msm_gem_put_iova(a5xx_gpu->pm4_bo, gpu->id);
 519                 drm_gem_object_unreference_unlocked(a5xx_gpu->pm4_bo);
 520         }
 521
 522         if (a5xx_gpu->pfp_bo) {
 523                 if (a5xx_gpu->pfp_iova)
 524                         msm_gem_put_iova(a5xx_gpu->pfp_bo, gpu->id);
 525                 drm_gem_object_unreference_unlocked(a5xx_gpu->pfp_bo);
 526         }
 527
 528         adreno_gpu_cleanup(adreno_gpu);
 529         kfree(a5xx_gpu);
 530 }
 531
 532 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
 533 {
 534         if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
 535                 return false;
 536
 537         /*
 538          * Nearly every abnormality ends up pausing the GPU and triggering a
 539          * fault so we can safely just watch for this one interrupt to fire
 540          */
 541         return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
 542                 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
 543 }
 544
 545 static bool a5xx_idle(struct msm_gpu *gpu)
 546 {
 547         /* wait for CP to drain ringbuffer: */
 548         if (!adreno_idle(gpu))
 549                 return false;
 550
 551         if (spin_until(_a5xx_check_idle(gpu))) {
 552                 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X\n",
 553                         gpu->name, __builtin_return_address(0),
 554                         gpu_read(gpu, REG_A5XX_RBBM_STATUS),
 555                         gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS));
 556
 557                 return false;
 558         }
 559
 560         return true;
 561 }
 562
 563 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
 564 {
 565         u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
 566
 567         if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
 568                 u32 val;
 569
 570                 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
 571
 572                 /*
 573                  * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
 574                  * read it twice
 575                  */
 576
 577                 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
 578                 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
 579
 580                 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
 581                         val);
 582         }
 583
 584         if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
 585                 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
 586                         gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
 587
 588         if (status & A5XX_CP_INT_CP_DMA_ERROR)
 589                 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
 590
 591         if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
 592                 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
 593
 594                 dev_err_ratelimited(gpu->dev->dev,
 595                         "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
 596                         val & (1 << 24) ? "WRITE" : "READ",
 597                         (val & 0xFFFFF) >> 2, val);
 598         }
 599
 600         if (status & A5XX_CP_INT_CP_AHB_ERROR) {
 601                 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
 602                 const char *access[16] = { "reserved", "reserved",
 603                         "timestamp lo", "timestamp hi", "pfp read", "pfp write",
 604                         "", "", "me read", "me write", "", "", "crashdump read",
 605                         "crashdump write" };
 606
 607                 dev_err_ratelimited(gpu->dev->dev,
 608                         "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
 609                         status & 0xFFFFF, access[(status >> 24) & 0xF],
 610                         (status & (1 << 31)), status);
 611         }
 612 }
 613
 614 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu)
 615 {
 616         u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
 617
 618         if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
 619                 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
 620
 621                 dev_err_ratelimited(gpu->dev->dev,
 622                         "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
 623                         val & (1 << 28) ? "WRITE" : "READ",
 624                         (val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
 625                         (val >> 24) & 0xF);
 626
 627                 /* Clear the error */
 628                 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
 629         }
 630
 631         if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
 632                 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
 633
 634         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
 635                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
 636                         gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
 637
 638         if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
 639                 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
 640                         gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
 641
 642         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
 643                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
 644                         gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
 645
 646         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
 647                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
 648
 649         if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
 650                 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
 651 }
 652
 653 static void a5xx_uche_err_irq(struct msm_gpu *gpu)
 654 {
 655         uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
 656
 657         addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
 658
 659         dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
 660                 addr);
 661 }
 662
 663 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
 664 {
 665         dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
 666 }
 667
 668 #define RBBM_ERROR_MASK \
 669         (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
 670         A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
 671         A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
 672         A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
 673         A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
 674         A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
 675
 676 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
 677 {
 678         u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
 679
 680         gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, status);
 681
 682         if (status & RBBM_ERROR_MASK)
 683                 a5xx_rbbm_err_irq(gpu);
 684
 685         if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
 686                 a5xx_cp_err_irq(gpu);
 687
 688         if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
 689                 a5xx_uche_err_irq(gpu);
 690
 691         if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
 692                 a5xx_gpmu_err_irq(gpu);
 693
 694         if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
 695                 msm_gpu_retire(gpu);
 696
 697         return IRQ_HANDLED;
 698 }
 699
 700 static const u32 a5xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
 701         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A5XX_CP_RB_BASE),
 702         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A5XX_CP_RB_BASE_HI),
 703         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A5XX_CP_RB_RPTR_ADDR),
 704         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
 705                 REG_A5XX_CP_RB_RPTR_ADDR_HI),
 706         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A5XX_CP_RB_RPTR),
 707         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A5XX_CP_RB_WPTR),
 708         REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A5XX_CP_RB_CNTL),
 709 };
 710
 711 static const u32 a5xx_registers[] = {
 712         0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
 713         0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
 714         0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
 715         0x04E0, 0x0533, 0x0540, 0x0555, 0xF400, 0xF400, 0xF800, 0xF807,
 716         0x0800, 0x081A, 0x081F, 0x0841, 0x0860, 0x0860, 0x0880, 0x08A0,
 717         0x0B00, 0x0B12, 0x0B15, 0x0B28, 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD,
 718         0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 0x0C60, 0x0C61, 0x0C80, 0x0C82,
 719         0x0C84, 0x0C85, 0x0C90, 0x0C98, 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2,
 720         0x2180, 0x2185, 0x2580, 0x2585, 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7,
 721         0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8,
 722         0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, 0x2100, 0x211E, 0x2140, 0x2145,
 723         0x2500, 0x251E, 0x2540, 0x2545, 0x0D10, 0x0D17, 0x0D20, 0x0D23,
 724         0x0D30, 0x0D30, 0x20C0, 0x20C0, 0x24C0, 0x24C0, 0x0E40, 0x0E43,
 725         0x0E4A, 0x0E4A, 0x0E50, 0x0E57, 0x0E60, 0x0E7C, 0x0E80, 0x0E8E,
 726         0x0E90, 0x0E96, 0x0EA0, 0x0EA8, 0x0EB0, 0x0EB2, 0xE140, 0xE147,
 727         0xE150, 0xE187, 0xE1A0, 0xE1A9, 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7,
 728         0xE1D0, 0xE1D1, 0xE200, 0xE201, 0xE210, 0xE21C, 0xE240, 0xE268,
 729         0xE000, 0xE006, 0xE010, 0xE09A, 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB,
 730         0xE100, 0xE105, 0xE380, 0xE38F, 0xE3B0, 0xE3B0, 0xE400, 0xE405,
 731         0xE408, 0xE4E9, 0xE4F0, 0xE4F0, 0xE280, 0xE280, 0xE282, 0xE2A3,
 732         0xE2A5, 0xE2C2, 0xE940, 0xE947, 0xE950, 0xE987, 0xE9A0, 0xE9A9,
 733         0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7, 0xE9D0, 0xE9D1, 0xEA00, 0xEA01,
 734         0xEA10, 0xEA1C, 0xEA40, 0xEA68, 0xE800, 0xE806, 0xE810, 0xE89A,
 735         0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, 0xE900, 0xE905, 0xEB80, 0xEB8F,
 736         0xEBB0, 0xEBB0, 0xEC00, 0xEC05, 0xEC08, 0xECE9, 0xECF0, 0xECF0,
 737         0xEA80, 0xEA80, 0xEA82, 0xEAA3, 0xEAA5, 0xEAC2, 0xA800, 0xA8FF,
 738         0xAC60, 0xAC60, 0xB000, 0xB97F, 0xB9A0, 0xB9BF,
 739         ~0
 740 };
 741
 742 static void a5xx_dump(struct msm_gpu *gpu)
 743 {
 744         dev_info(gpu->dev->dev, "status:   %08x\n",
 745                 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
 746         adreno_dump(gpu);
 747 }
 748
 749 static int a5xx_pm_resume(struct msm_gpu *gpu)
 750 {
 751         return msm_gpu_pm_resume(gpu);
 752 }
 753
 754 static int a5xx_pm_suspend(struct msm_gpu *gpu)
 755 {
 756         return msm_gpu_pm_suspend(gpu);
 757 }
 758
 759 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 760 {
 761         *value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,
 762                 REG_A5XX_RBBM_PERFCTR_CP_0_HI);
 763
 764         return 0;
 765 }
 766
 767 #ifdef CONFIG_DEBUG_FS
 768 static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
 769 {
 770         gpu->funcs->pm_resume(gpu);
 771
 772         seq_printf(m, "status:   %08x\n",
 773                         gpu_read(gpu, REG_A5XX_RBBM_STATUS));
 774         gpu->funcs->pm_suspend(gpu);
 775
 776         adreno_show(gpu, m);
 777 }
 778 #endif
 779
 780 static const struct adreno_gpu_funcs funcs = {
 781         .base = {
 782                 .get_param = adreno_get_param,
 783                 .hw_init = a5xx_hw_init,
 784                 .pm_suspend = a5xx_pm_suspend,
 785                 .pm_resume = a5xx_pm_resume,
 786                 .recover = a5xx_recover,
 787                 .last_fence = adreno_last_fence,
 788                 .submit = a5xx_submit,
 789                 .flush = adreno_flush,
 790                 .idle = a5xx_idle,
 791                 .irq = a5xx_irq,
 792                 .destroy = a5xx_destroy,
 793                 .show = a5xx_show,
 794         },
 795         .get_timestamp = a5xx_get_timestamp,
 796 };
 797
 798 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
 799 {
 800         struct msm_drm_private *priv = dev->dev_private;
 801         struct platform_device *pdev = priv->gpu_pdev;
 802         struct a5xx_gpu *a5xx_gpu = NULL;
 803         struct adreno_gpu *adreno_gpu;
 804         struct msm_gpu *gpu;
 805         int ret;
 806
 807         if (!pdev) {
 808                 dev_err(dev->dev, "No A5XX device is defined\n");
 809                 return ERR_PTR(-ENXIO);
 810         }
 811
 812         a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
 813         if (!a5xx_gpu)
 814                 return ERR_PTR(-ENOMEM);
 815
 816         adreno_gpu = &a5xx_gpu->base;
 817         gpu = &adreno_gpu->base;
 818
 819         a5xx_gpu->pdev = pdev;
 820         adreno_gpu->registers = a5xx_registers;
 821         adreno_gpu->reg_offsets = a5xx_register_offsets;
 822
 823         ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs);
 824         if (ret) {
 825                 a5xx_destroy(&(a5xx_gpu->base.base));
 826                 return ERR_PTR(ret);
 827         }
 828
 829         return gpu;
 830 }