1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
8 #include "a6xx_gmu.xml.h"
10 static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
12 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
13 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
15 /* Check that the GMU is idle */
16 if (!a6xx_gmu_isidle(&a6xx_gpu->gmu))
19 /* Check tha the CX master is idle */
20 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &
21 ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
24 return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &
25 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
28 bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
30 /* wait for CP to drain ringbuffer: */
31 if (!adreno_idle(gpu, ring))
34 if (spin_until(_a6xx_check_idle(gpu))) {
35 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
36 gpu->name, __builtin_return_address(0),
37 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
38 gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),
39 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
40 gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
47 static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
52 spin_lock_irqsave(&ring->lock, flags);
54 /* Copy the shadow to the actual register */
55 ring->cur = ring->next;
57 /* Make sure to wrap wptr if we need to */
58 wptr = get_wptr(ring);
60 spin_unlock_irqrestore(&ring->lock, flags);
62 /* Make sure everything is posted before making a decision */
65 gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
68 static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
69 struct msm_file_private *ctx)
71 struct msm_drm_private *priv = gpu->dev->dev_private;
72 struct msm_ringbuffer *ring = submit->ring;
75 /* Invalidate CCU depth and color */
76 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
77 OUT_RING(ring, PC_CCU_INVALIDATE_DEPTH);
79 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
80 OUT_RING(ring, PC_CCU_INVALIDATE_COLOR);
82 /* Submit the commands */
83 for (i = 0; i < submit->nr_cmds; i++) {
84 switch (submit->cmd[i].type) {
85 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
87 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
88 if (priv->lastctx == ctx)
90 case MSM_SUBMIT_CMD_BUF:
91 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
92 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
93 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
94 OUT_RING(ring, submit->cmd[i].size);
99 /* Write the fence to the scratch register */
100 OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
101 OUT_RING(ring, submit->seqno);
104 * Execute a CACHE_FLUSH_TS event. This will ensure that the
105 * timestamp is written to the memory and then triggers the interrupt
107 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
108 OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
109 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
110 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
111 OUT_RING(ring, submit->seqno);
113 a6xx_flush(gpu, ring);
116 static const struct {
120 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
121 {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222},
122 {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222},
123 {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222},
124 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220},
125 {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220},
126 {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220},
127 {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220},
128 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
129 {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
130 {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
131 {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
132 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
133 {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf},
134 {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf},
135 {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf},
136 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
137 {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
138 {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222},
139 {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222},
140 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
141 {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
142 {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
143 {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
144 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
145 {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
146 {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222},
147 {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222},
148 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
149 {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
150 {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222},
151 {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222},
152 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
153 {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
154 {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
155 {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
156 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
157 {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
158 {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
159 {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
160 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
161 {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
162 {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777},
163 {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777},
164 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
165 {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
166 {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777},
167 {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777},
168 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
169 {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
170 {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
171 {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
172 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
173 {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
174 {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
175 {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
176 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
177 {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
178 {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111},
179 {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111},
180 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
181 {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
182 {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111},
183 {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111},
184 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
185 {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
186 {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
187 {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
188 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
189 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
190 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
191 {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
192 {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
193 {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
194 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
195 {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222},
196 {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222},
197 {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222},
198 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
199 {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
200 {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
201 {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
202 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
203 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00},
204 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00},
205 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00},
206 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
207 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
208 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
209 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
210 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
211 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
212 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
213 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
214 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
215 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
216 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
217 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
218 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
219 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
220 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
221 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
222 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
223 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
224 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}
227 static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
229 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
230 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
231 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
235 val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);
237 /* Don't re-program the registers if they are already correct */
238 if ((!state && !val) || (state && (val == 0x8aa8aa02)))
241 /* Disable SP clock before programming HWCG registers */
242 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0);
244 for (i = 0; i < ARRAY_SIZE(a6xx_hwcg); i++)
245 gpu_write(gpu, a6xx_hwcg[i].offset,
246 state ? a6xx_hwcg[i].value : 0);
248 /* Enable SP clock */
249 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1);
251 gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? 0x8aa8aa02 : 0);
254 static int a6xx_cp_init(struct msm_gpu *gpu)
256 struct msm_ringbuffer *ring = gpu->rb[0];
258 OUT_PKT7(ring, CP_ME_INIT, 8);
260 OUT_RING(ring, 0x0000002f);
262 /* Enable multiple hardware contexts */
263 OUT_RING(ring, 0x00000003);
265 /* Enable error detection */
266 OUT_RING(ring, 0x20000000);
268 /* Don't enable header dump */
269 OUT_RING(ring, 0x00000000);
270 OUT_RING(ring, 0x00000000);
272 /* No workarounds enabled */
273 OUT_RING(ring, 0x00000000);
275 /* Pad rest of the cmds with 0's */
276 OUT_RING(ring, 0x00000000);
277 OUT_RING(ring, 0x00000000);
279 a6xx_flush(gpu, ring);
280 return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
283 static int a6xx_ucode_init(struct msm_gpu *gpu)
285 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
286 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
288 if (!a6xx_gpu->sqe_bo) {
289 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,
290 adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova);
292 if (IS_ERR(a6xx_gpu->sqe_bo)) {
293 int ret = PTR_ERR(a6xx_gpu->sqe_bo);
295 a6xx_gpu->sqe_bo = NULL;
296 DRM_DEV_ERROR(&gpu->pdev->dev,
297 "Could not allocate SQE ucode: %d\n", ret);
303 gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE_LO,
304 REG_A6XX_CP_SQE_INSTR_BASE_HI, a6xx_gpu->sqe_iova);
309 #define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
310 A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
311 A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
312 A6XX_RBBM_INT_0_MASK_CP_IB2 | \
313 A6XX_RBBM_INT_0_MASK_CP_IB1 | \
314 A6XX_RBBM_INT_0_MASK_CP_RB | \
315 A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
316 A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
317 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
318 A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
319 A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
321 static int a6xx_hw_init(struct msm_gpu *gpu)
323 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
324 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
327 /* Make sure the GMU keeps the GPU on while we set it up */
328 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
330 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
333 * Disable the trusted memory range - we don't actually supported secure
334 * memory rendering at this point in time and we don't want to block off
335 * part of the virtual memory space.
337 gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
338 REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
339 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
341 /* enable hardware clockgating */
342 a6xx_set_hwcg(gpu, true);
345 gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
346 gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
348 /* Make all blocks contribute to the GPU BUSY perf counter */
349 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
351 /* Disable L2 bypass in the UCHE */
352 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0);
353 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff);
354 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000);
355 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff);
356 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);
357 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);
359 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
360 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,
361 REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000);
363 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,
364 REG_A6XX_UCHE_GMEM_RANGE_MAX_HI,
365 0x00100000 + adreno_gpu->gmem - 1);
367 gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
368 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
370 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
371 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
373 /* Setting the mem pool size */
374 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
376 /* Setting the primFifo thresholds default values */
377 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11));
379 /* Set the AHB default slave response to "ERROR" */
380 gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
382 /* Turn on performance counters */
383 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);
385 /* Select CP0 to always count cycles */
386 gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
388 /* FIXME: not sure if this should live here or in a6xx_gmu.c */
389 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK,
391 gmu_rmw(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0,
393 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE,
396 gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, 2 << 1);
397 gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, 2 << 1);
398 gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, 2 << 1);
399 gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, 2 << 21);
401 /* Enable fault detection */
402 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL,
403 (1 << 30) | 0x1fffff);
405 gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1);
407 /* Protect registers from the CP */
408 gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003);
410 gpu_write(gpu, REG_A6XX_CP_PROTECT(0),
411 A6XX_PROTECT_RDONLY(0x600, 0x51));
412 gpu_write(gpu, REG_A6XX_CP_PROTECT(1), A6XX_PROTECT_RW(0xae50, 0x2));
413 gpu_write(gpu, REG_A6XX_CP_PROTECT(2), A6XX_PROTECT_RW(0x9624, 0x13));
414 gpu_write(gpu, REG_A6XX_CP_PROTECT(3), A6XX_PROTECT_RW(0x8630, 0x8));
415 gpu_write(gpu, REG_A6XX_CP_PROTECT(4), A6XX_PROTECT_RW(0x9e70, 0x1));
416 gpu_write(gpu, REG_A6XX_CP_PROTECT(5), A6XX_PROTECT_RW(0x9e78, 0x187));
417 gpu_write(gpu, REG_A6XX_CP_PROTECT(6), A6XX_PROTECT_RW(0xf000, 0x810));
418 gpu_write(gpu, REG_A6XX_CP_PROTECT(7),
419 A6XX_PROTECT_RDONLY(0xfc00, 0x3));
420 gpu_write(gpu, REG_A6XX_CP_PROTECT(8), A6XX_PROTECT_RW(0x50e, 0x0));
421 gpu_write(gpu, REG_A6XX_CP_PROTECT(9), A6XX_PROTECT_RDONLY(0x50f, 0x0));
422 gpu_write(gpu, REG_A6XX_CP_PROTECT(10), A6XX_PROTECT_RW(0x510, 0x0));
423 gpu_write(gpu, REG_A6XX_CP_PROTECT(11),
424 A6XX_PROTECT_RDONLY(0x0, 0x4f9));
425 gpu_write(gpu, REG_A6XX_CP_PROTECT(12),
426 A6XX_PROTECT_RDONLY(0x501, 0xa));
427 gpu_write(gpu, REG_A6XX_CP_PROTECT(13),
428 A6XX_PROTECT_RDONLY(0x511, 0x44));
429 gpu_write(gpu, REG_A6XX_CP_PROTECT(14), A6XX_PROTECT_RW(0xe00, 0xe));
430 gpu_write(gpu, REG_A6XX_CP_PROTECT(15), A6XX_PROTECT_RW(0x8e00, 0x0));
431 gpu_write(gpu, REG_A6XX_CP_PROTECT(16), A6XX_PROTECT_RW(0x8e50, 0xf));
432 gpu_write(gpu, REG_A6XX_CP_PROTECT(17), A6XX_PROTECT_RW(0xbe02, 0x0));
433 gpu_write(gpu, REG_A6XX_CP_PROTECT(18),
434 A6XX_PROTECT_RW(0xbe20, 0x11f3));
435 gpu_write(gpu, REG_A6XX_CP_PROTECT(19), A6XX_PROTECT_RW(0x800, 0x82));
436 gpu_write(gpu, REG_A6XX_CP_PROTECT(20), A6XX_PROTECT_RW(0x8a0, 0x8));
437 gpu_write(gpu, REG_A6XX_CP_PROTECT(21), A6XX_PROTECT_RW(0x8ab, 0x19));
438 gpu_write(gpu, REG_A6XX_CP_PROTECT(22), A6XX_PROTECT_RW(0x900, 0x4d));
439 gpu_write(gpu, REG_A6XX_CP_PROTECT(23), A6XX_PROTECT_RW(0x98d, 0x76));
440 gpu_write(gpu, REG_A6XX_CP_PROTECT(24),
441 A6XX_PROTECT_RDONLY(0x8d0, 0x23));
442 gpu_write(gpu, REG_A6XX_CP_PROTECT(25),
443 A6XX_PROTECT_RDONLY(0x980, 0x4));
444 gpu_write(gpu, REG_A6XX_CP_PROTECT(26), A6XX_PROTECT_RW(0xa630, 0x0));
446 /* Enable interrupts */
447 gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK);
449 ret = adreno_hw_init(gpu);
453 ret = a6xx_ucode_init(gpu);
457 /* Always come up on rb 0 */
458 a6xx_gpu->cur_ring = gpu->rb[0];
460 /* Enable the SQE_to start the CP engine */
461 gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);
463 ret = a6xx_cp_init(gpu);
467 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
471 * Tell the GMU that we are done touching the GPU and it can start power
474 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
476 /* Take the GMU out of its special boot mode */
477 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
482 static void a6xx_dump(struct msm_gpu *gpu)
484 dev_info(&gpu->pdev->dev, "status: %08x\n",
485 gpu_read(gpu, REG_A6XX_RBBM_STATUS));
489 #define VBIF_RESET_ACK_TIMEOUT 100
490 #define VBIF_RESET_ACK_MASK 0x00f0
492 static void a6xx_recover(struct msm_gpu *gpu)
494 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
495 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
498 adreno_dump_info(gpu);
500 for (i = 0; i < 8; i++)
501 dev_info(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,
502 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));
508 * Turn off keep alive that might have been enabled by the hang
511 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
513 gpu->funcs->pm_suspend(gpu);
514 gpu->funcs->pm_resume(gpu);
516 msm_gpu_hw_init(gpu);
519 static int a6xx_fault_handler(void *arg, unsigned long iova, int flags)
521 struct msm_gpu *gpu = arg;
523 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
525 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
526 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
527 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
528 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
533 static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
535 u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);
537 if (status & A6XX_CP_INT_CP_OPCODE_ERROR) {
540 gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);
541 val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);
542 dev_err_ratelimited(&gpu->pdev->dev,
543 "CP | opcode error | possible opcode=0x%8.8X\n",
547 if (status & A6XX_CP_INT_CP_UCODE_ERROR)
548 dev_err_ratelimited(&gpu->pdev->dev,
549 "CP ucode error interrupt\n");
551 if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR)
552 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",
553 gpu_read(gpu, REG_A6XX_CP_HW_FAULT));
555 if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
556 u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);
558 dev_err_ratelimited(&gpu->pdev->dev,
559 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
560 val & (1 << 20) ? "READ" : "WRITE",
561 (val & 0x3ffff), val);
564 if (status & A6XX_CP_INT_CP_AHB_ERROR)
565 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");
567 if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR)
568 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");
570 if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR)
571 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");
575 static void a6xx_fault_detect_irq(struct msm_gpu *gpu)
577 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
578 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
579 struct drm_device *dev = gpu->dev;
580 struct msm_drm_private *priv = dev->dev_private;
581 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
584 * Force the GPU to stay on until after we finish
585 * collecting information
587 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);
589 DRM_DEV_ERROR(&gpu->pdev->dev,
590 "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
591 ring ? ring->id : -1, ring ? ring->seqno : 0,
592 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
593 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
594 gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
595 gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI),
596 gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
597 gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI),
598 gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));
600 /* Turn off the hangcheck timer to keep it from bothering us */
601 del_timer(&gpu->hangcheck_timer);
603 queue_work(priv->wq, &gpu->recover_work);
606 static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
608 u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);
610 gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
612 if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
613 a6xx_fault_detect_irq(gpu);
615 if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR)
616 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");
618 if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
619 a6xx_cp_hw_err_irq(gpu);
621 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
622 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
624 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
625 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
627 if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
628 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
630 if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
636 static const u32 a6xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
637 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A6XX_CP_RB_BASE),
638 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A6XX_CP_RB_BASE_HI),
639 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR,
640 REG_A6XX_CP_RB_RPTR_ADDR_LO),
641 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
642 REG_A6XX_CP_RB_RPTR_ADDR_HI),
643 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A6XX_CP_RB_RPTR),
644 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A6XX_CP_RB_WPTR),
645 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A6XX_CP_RB_CNTL),
648 static const u32 a6xx_registers[] = {
649 0x0000, 0x0002, 0x0010, 0x0010, 0x0012, 0x0012, 0x0018, 0x001b,
650 0x001e, 0x0032, 0x0038, 0x003c, 0x0042, 0x0042, 0x0044, 0x0044,
651 0x0047, 0x0047, 0x0056, 0x0056, 0x00ad, 0x00ae, 0x00b0, 0x00fb,
652 0x0100, 0x011d, 0x0200, 0x020d, 0x0210, 0x0213, 0x0218, 0x023d,
653 0x0400, 0x04f9, 0x0500, 0x0500, 0x0505, 0x050b, 0x050e, 0x0511,
654 0x0533, 0x0533, 0x0540, 0x0555, 0x0800, 0x0808, 0x0810, 0x0813,
655 0x0820, 0x0821, 0x0823, 0x0827, 0x0830, 0x0833, 0x0840, 0x0843,
656 0x084f, 0x086f, 0x0880, 0x088a, 0x08a0, 0x08ab, 0x08c0, 0x08c4,
657 0x08d0, 0x08dd, 0x08f0, 0x08f3, 0x0900, 0x0903, 0x0908, 0x0911,
658 0x0928, 0x093e, 0x0942, 0x094d, 0x0980, 0x0984, 0x098d, 0x0996,
659 0x0998, 0x099e, 0x09a0, 0x09a6, 0x09a8, 0x09ae, 0x09b0, 0x09b1,
660 0x09c2, 0x09c8, 0x0a00, 0x0a03, 0x0c00, 0x0c04, 0x0c06, 0x0c06,
661 0x0c10, 0x0cd9, 0x0e00, 0x0e0e, 0x0e10, 0x0e13, 0x0e17, 0x0e19,
662 0x0e1c, 0x0e2b, 0x0e30, 0x0e32, 0x0e38, 0x0e39, 0x8600, 0x8601,
663 0x8610, 0x861b, 0x8620, 0x8620, 0x8628, 0x862b, 0x8630, 0x8637,
664 0x8e01, 0x8e01, 0x8e04, 0x8e05, 0x8e07, 0x8e08, 0x8e0c, 0x8e0c,
665 0x8e10, 0x8e1c, 0x8e20, 0x8e25, 0x8e28, 0x8e28, 0x8e2c, 0x8e2f,
666 0x8e3b, 0x8e3e, 0x8e40, 0x8e43, 0x8e50, 0x8e5e, 0x8e70, 0x8e77,
667 0x9600, 0x9604, 0x9624, 0x9637, 0x9e00, 0x9e01, 0x9e03, 0x9e0e,
668 0x9e11, 0x9e16, 0x9e19, 0x9e19, 0x9e1c, 0x9e1c, 0x9e20, 0x9e23,
669 0x9e30, 0x9e31, 0x9e34, 0x9e34, 0x9e70, 0x9e72, 0x9e78, 0x9e79,
670 0x9e80, 0x9fff, 0xa600, 0xa601, 0xa603, 0xa603, 0xa60a, 0xa60a,
671 0xa610, 0xa617, 0xa630, 0xa630,
675 static int a6xx_pm_resume(struct msm_gpu *gpu)
677 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
678 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
681 ret = a6xx_gmu_resume(a6xx_gpu);
683 gpu->needs_hw_init = true;
688 static int a6xx_pm_suspend(struct msm_gpu *gpu)
690 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
691 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
694 * Make sure the GMU is idle before continuing (because some transitions
697 a6xx_gmu_wait_for_idle(a6xx_gpu);
699 /* Clear the VBIF pipe before shutting down */
700 /* FIXME: This accesses the GPU - do we need to make sure it is on? */
701 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);
702 spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & 0xf) == 0xf);
703 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
705 return a6xx_gmu_stop(a6xx_gpu);
708 static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
710 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
711 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
713 /* Force the GPU power on so we can read this register */
714 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
716 *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
717 REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
719 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
723 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
724 static void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
725 struct drm_printer *p)
727 adreno_show(gpu, state, p);
731 static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)
733 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
734 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
736 return a6xx_gpu->cur_ring;
739 static void a6xx_destroy(struct msm_gpu *gpu)
741 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
742 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
744 if (a6xx_gpu->sqe_bo) {
745 if (a6xx_gpu->sqe_iova)
746 msm_gem_put_iova(a6xx_gpu->sqe_bo, gpu->aspace);
747 drm_gem_object_unreference_unlocked(a6xx_gpu->sqe_bo);
750 a6xx_gmu_remove(a6xx_gpu);
752 adreno_gpu_cleanup(adreno_gpu);
756 static const struct adreno_gpu_funcs funcs = {
758 .get_param = adreno_get_param,
759 .hw_init = a6xx_hw_init,
760 .pm_suspend = a6xx_pm_suspend,
761 .pm_resume = a6xx_pm_resume,
762 .recover = a6xx_recover,
763 .submit = a6xx_submit,
765 .active_ring = a6xx_active_ring,
767 .destroy = a6xx_destroy,
768 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
772 .get_timestamp = a6xx_get_timestamp,
775 struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
777 struct msm_drm_private *priv = dev->dev_private;
778 struct platform_device *pdev = priv->gpu_pdev;
779 struct device_node *node;
780 struct a6xx_gpu *a6xx_gpu;
781 struct adreno_gpu *adreno_gpu;
785 a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL);
787 return ERR_PTR(-ENOMEM);
789 adreno_gpu = &a6xx_gpu->base;
790 gpu = &adreno_gpu->base;
792 adreno_gpu->registers = a6xx_registers;
793 adreno_gpu->reg_offsets = a6xx_register_offsets;
795 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
797 a6xx_destroy(&(a6xx_gpu->base.base));
801 /* Check if there is a GMU phandle and set it up */
802 node = of_parse_phandle(pdev->dev.of_node, "gmu", 0);
804 /* FIXME: How do we gracefully handle this? */
807 ret = a6xx_gmu_probe(a6xx_gpu, node);
809 a6xx_destroy(&(a6xx_gpu->base.base));
814 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,