2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
31 #include <drm/radeon_drm.h>
34 #include "si_blit_shaders.h"
35 #include "clearstate_si.h"
36 #include "radeon_ucode.h"
41 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
42 static void si_pcie_gen3_enable(struct radeon_device *rdev);
43 static void si_program_aspm(struct radeon_device *rdev);
44 extern void sumo_rlc_fini(struct radeon_device *rdev);
45 extern int sumo_rlc_init(struct radeon_device *rdev);
46 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
47 extern void r600_ih_ring_fini(struct radeon_device *rdev);
48 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
49 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
50 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
51 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
52 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
53 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
54 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
56 static void si_init_pg(struct radeon_device *rdev);
57 static void si_init_cg(struct radeon_device *rdev);
58 static void si_fini_pg(struct radeon_device *rdev);
59 static void si_fini_cg(struct radeon_device *rdev);
60 static void si_rlc_stop(struct radeon_device *rdev);
62 static const u32 verde_rlc_save_restore_register_list[] =
64 (0x8000 << 16) | (0x98f4 >> 2),
66 (0x8040 << 16) | (0x98f4 >> 2),
68 (0x8000 << 16) | (0xe80 >> 2),
70 (0x8040 << 16) | (0xe80 >> 2),
72 (0x8000 << 16) | (0x89bc >> 2),
74 (0x8040 << 16) | (0x89bc >> 2),
76 (0x8000 << 16) | (0x8c1c >> 2),
78 (0x8040 << 16) | (0x8c1c >> 2),
80 (0x9c00 << 16) | (0x98f0 >> 2),
82 (0x9c00 << 16) | (0xe7c >> 2),
84 (0x8000 << 16) | (0x9148 >> 2),
86 (0x8040 << 16) | (0x9148 >> 2),
88 (0x9c00 << 16) | (0x9150 >> 2),
90 (0x9c00 << 16) | (0x897c >> 2),
92 (0x9c00 << 16) | (0x8d8c >> 2),
94 (0x9c00 << 16) | (0xac54 >> 2),
97 (0x9c00 << 16) | (0x98f8 >> 2),
99 (0x9c00 << 16) | (0x9910 >> 2),
101 (0x9c00 << 16) | (0x9914 >> 2),
103 (0x9c00 << 16) | (0x9918 >> 2),
105 (0x9c00 << 16) | (0x991c >> 2),
107 (0x9c00 << 16) | (0x9920 >> 2),
109 (0x9c00 << 16) | (0x9924 >> 2),
111 (0x9c00 << 16) | (0x9928 >> 2),
113 (0x9c00 << 16) | (0x992c >> 2),
115 (0x9c00 << 16) | (0x9930 >> 2),
117 (0x9c00 << 16) | (0x9934 >> 2),
119 (0x9c00 << 16) | (0x9938 >> 2),
121 (0x9c00 << 16) | (0x993c >> 2),
123 (0x9c00 << 16) | (0x9940 >> 2),
125 (0x9c00 << 16) | (0x9944 >> 2),
127 (0x9c00 << 16) | (0x9948 >> 2),
129 (0x9c00 << 16) | (0x994c >> 2),
131 (0x9c00 << 16) | (0x9950 >> 2),
133 (0x9c00 << 16) | (0x9954 >> 2),
135 (0x9c00 << 16) | (0x9958 >> 2),
137 (0x9c00 << 16) | (0x995c >> 2),
139 (0x9c00 << 16) | (0x9960 >> 2),
141 (0x9c00 << 16) | (0x9964 >> 2),
143 (0x9c00 << 16) | (0x9968 >> 2),
145 (0x9c00 << 16) | (0x996c >> 2),
147 (0x9c00 << 16) | (0x9970 >> 2),
149 (0x9c00 << 16) | (0x9974 >> 2),
151 (0x9c00 << 16) | (0x9978 >> 2),
153 (0x9c00 << 16) | (0x997c >> 2),
155 (0x9c00 << 16) | (0x9980 >> 2),
157 (0x9c00 << 16) | (0x9984 >> 2),
159 (0x9c00 << 16) | (0x9988 >> 2),
161 (0x9c00 << 16) | (0x998c >> 2),
163 (0x9c00 << 16) | (0x8c00 >> 2),
165 (0x9c00 << 16) | (0x8c14 >> 2),
167 (0x9c00 << 16) | (0x8c04 >> 2),
169 (0x9c00 << 16) | (0x8c08 >> 2),
171 (0x8000 << 16) | (0x9b7c >> 2),
173 (0x8040 << 16) | (0x9b7c >> 2),
175 (0x8000 << 16) | (0xe84 >> 2),
177 (0x8040 << 16) | (0xe84 >> 2),
179 (0x8000 << 16) | (0x89c0 >> 2),
181 (0x8040 << 16) | (0x89c0 >> 2),
183 (0x8000 << 16) | (0x914c >> 2),
185 (0x8040 << 16) | (0x914c >> 2),
187 (0x8000 << 16) | (0x8c20 >> 2),
189 (0x8040 << 16) | (0x8c20 >> 2),
191 (0x8000 << 16) | (0x9354 >> 2),
193 (0x8040 << 16) | (0x9354 >> 2),
195 (0x9c00 << 16) | (0x9060 >> 2),
197 (0x9c00 << 16) | (0x9364 >> 2),
199 (0x9c00 << 16) | (0x9100 >> 2),
201 (0x9c00 << 16) | (0x913c >> 2),
203 (0x8000 << 16) | (0x90e0 >> 2),
205 (0x8000 << 16) | (0x90e4 >> 2),
207 (0x8000 << 16) | (0x90e8 >> 2),
209 (0x8040 << 16) | (0x90e0 >> 2),
211 (0x8040 << 16) | (0x90e4 >> 2),
213 (0x8040 << 16) | (0x90e8 >> 2),
215 (0x9c00 << 16) | (0x8bcc >> 2),
217 (0x9c00 << 16) | (0x8b24 >> 2),
219 (0x9c00 << 16) | (0x88c4 >> 2),
221 (0x9c00 << 16) | (0x8e50 >> 2),
223 (0x9c00 << 16) | (0x8c0c >> 2),
225 (0x9c00 << 16) | (0x8e58 >> 2),
227 (0x9c00 << 16) | (0x8e5c >> 2),
229 (0x9c00 << 16) | (0x9508 >> 2),
231 (0x9c00 << 16) | (0x950c >> 2),
233 (0x9c00 << 16) | (0x9494 >> 2),
235 (0x9c00 << 16) | (0xac0c >> 2),
237 (0x9c00 << 16) | (0xac10 >> 2),
239 (0x9c00 << 16) | (0xac14 >> 2),
241 (0x9c00 << 16) | (0xae00 >> 2),
243 (0x9c00 << 16) | (0xac08 >> 2),
245 (0x9c00 << 16) | (0x88d4 >> 2),
247 (0x9c00 << 16) | (0x88c8 >> 2),
249 (0x9c00 << 16) | (0x88cc >> 2),
251 (0x9c00 << 16) | (0x89b0 >> 2),
253 (0x9c00 << 16) | (0x8b10 >> 2),
255 (0x9c00 << 16) | (0x8a14 >> 2),
257 (0x9c00 << 16) | (0x9830 >> 2),
259 (0x9c00 << 16) | (0x9834 >> 2),
261 (0x9c00 << 16) | (0x9838 >> 2),
263 (0x9c00 << 16) | (0x9a10 >> 2),
265 (0x8000 << 16) | (0x9870 >> 2),
267 (0x8000 << 16) | (0x9874 >> 2),
269 (0x8001 << 16) | (0x9870 >> 2),
271 (0x8001 << 16) | (0x9874 >> 2),
273 (0x8040 << 16) | (0x9870 >> 2),
275 (0x8040 << 16) | (0x9874 >> 2),
277 (0x8041 << 16) | (0x9870 >> 2),
279 (0x8041 << 16) | (0x9874 >> 2),
284 static const u32 tahiti_golden_rlc_registers[] =
286 0xc424, 0xffffffff, 0x00601005,
287 0xc47c, 0xffffffff, 0x10104040,
288 0xc488, 0xffffffff, 0x0100000a,
289 0xc314, 0xffffffff, 0x00000800,
290 0xc30c, 0xffffffff, 0x800000f4,
291 0xf4a8, 0xffffffff, 0x00000000
294 static const u32 tahiti_golden_registers[] =
296 0x9a10, 0x00010000, 0x00018208,
297 0x9830, 0xffffffff, 0x00000000,
298 0x9834, 0xf00fffff, 0x00000400,
299 0x9838, 0x0002021c, 0x00020200,
300 0xc78, 0x00000080, 0x00000000,
301 0xd030, 0x000300c0, 0x00800040,
302 0xd830, 0x000300c0, 0x00800040,
303 0x5bb0, 0x000000f0, 0x00000070,
304 0x5bc0, 0x00200000, 0x50100000,
305 0x7030, 0x31000311, 0x00000011,
306 0x277c, 0x00000003, 0x000007ff,
307 0x240c, 0x000007ff, 0x00000000,
308 0x8a14, 0xf000001f, 0x00000007,
309 0x8b24, 0xffffffff, 0x00ffffff,
310 0x8b10, 0x0000ff0f, 0x00000000,
311 0x28a4c, 0x07ffffff, 0x4e000000,
312 0x28350, 0x3f3f3fff, 0x2a00126a,
313 0x30, 0x000000ff, 0x0040,
314 0x34, 0x00000040, 0x00004040,
315 0x9100, 0x07ffffff, 0x03000000,
316 0x8e88, 0x01ff1f3f, 0x00000000,
317 0x8e84, 0x01ff1f3f, 0x00000000,
318 0x9060, 0x0000007f, 0x00000020,
319 0x9508, 0x00010000, 0x00010000,
320 0xac14, 0x00000200, 0x000002fb,
321 0xac10, 0xffffffff, 0x0000543b,
322 0xac0c, 0xffffffff, 0xa9210876,
323 0x88d0, 0xffffffff, 0x000fff40,
324 0x88d4, 0x0000001f, 0x00000010,
325 0x1410, 0x20000000, 0x20fffed8,
326 0x15c0, 0x000c0fc0, 0x000c0400
329 static const u32 tahiti_golden_registers2[] =
331 0xc64, 0x00000001, 0x00000001
334 static const u32 pitcairn_golden_rlc_registers[] =
336 0xc424, 0xffffffff, 0x00601004,
337 0xc47c, 0xffffffff, 0x10102020,
338 0xc488, 0xffffffff, 0x01000020,
339 0xc314, 0xffffffff, 0x00000800,
340 0xc30c, 0xffffffff, 0x800000a4
343 static const u32 pitcairn_golden_registers[] =
345 0x9a10, 0x00010000, 0x00018208,
346 0x9830, 0xffffffff, 0x00000000,
347 0x9834, 0xf00fffff, 0x00000400,
348 0x9838, 0x0002021c, 0x00020200,
349 0xc78, 0x00000080, 0x00000000,
350 0xd030, 0x000300c0, 0x00800040,
351 0xd830, 0x000300c0, 0x00800040,
352 0x5bb0, 0x000000f0, 0x00000070,
353 0x5bc0, 0x00200000, 0x50100000,
354 0x7030, 0x31000311, 0x00000011,
355 0x2ae4, 0x00073ffe, 0x000022a2,
356 0x240c, 0x000007ff, 0x00000000,
357 0x8a14, 0xf000001f, 0x00000007,
358 0x8b24, 0xffffffff, 0x00ffffff,
359 0x8b10, 0x0000ff0f, 0x00000000,
360 0x28a4c, 0x07ffffff, 0x4e000000,
361 0x28350, 0x3f3f3fff, 0x2a00126a,
362 0x30, 0x000000ff, 0x0040,
363 0x34, 0x00000040, 0x00004040,
364 0x9100, 0x07ffffff, 0x03000000,
365 0x9060, 0x0000007f, 0x00000020,
366 0x9508, 0x00010000, 0x00010000,
367 0xac14, 0x000003ff, 0x000000f7,
368 0xac10, 0xffffffff, 0x00000000,
369 0xac0c, 0xffffffff, 0x32761054,
370 0x88d4, 0x0000001f, 0x00000010,
371 0x15c0, 0x000c0fc0, 0x000c0400
374 static const u32 verde_golden_rlc_registers[] =
376 0xc424, 0xffffffff, 0x033f1005,
377 0xc47c, 0xffffffff, 0x10808020,
378 0xc488, 0xffffffff, 0x00800008,
379 0xc314, 0xffffffff, 0x00001000,
380 0xc30c, 0xffffffff, 0x80010014
383 static const u32 verde_golden_registers[] =
385 0x9a10, 0x00010000, 0x00018208,
386 0x9830, 0xffffffff, 0x00000000,
387 0x9834, 0xf00fffff, 0x00000400,
388 0x9838, 0x0002021c, 0x00020200,
389 0xc78, 0x00000080, 0x00000000,
390 0xd030, 0x000300c0, 0x00800040,
391 0xd030, 0x000300c0, 0x00800040,
392 0xd830, 0x000300c0, 0x00800040,
393 0xd830, 0x000300c0, 0x00800040,
394 0x5bb0, 0x000000f0, 0x00000070,
395 0x5bc0, 0x00200000, 0x50100000,
396 0x7030, 0x31000311, 0x00000011,
397 0x2ae4, 0x00073ffe, 0x000022a2,
398 0x2ae4, 0x00073ffe, 0x000022a2,
399 0x2ae4, 0x00073ffe, 0x000022a2,
400 0x240c, 0x000007ff, 0x00000000,
401 0x240c, 0x000007ff, 0x00000000,
402 0x240c, 0x000007ff, 0x00000000,
403 0x8a14, 0xf000001f, 0x00000007,
404 0x8a14, 0xf000001f, 0x00000007,
405 0x8a14, 0xf000001f, 0x00000007,
406 0x8b24, 0xffffffff, 0x00ffffff,
407 0x8b10, 0x0000ff0f, 0x00000000,
408 0x28a4c, 0x07ffffff, 0x4e000000,
409 0x28350, 0x3f3f3fff, 0x0000124a,
410 0x28350, 0x3f3f3fff, 0x0000124a,
411 0x28350, 0x3f3f3fff, 0x0000124a,
412 0x30, 0x000000ff, 0x0040,
413 0x34, 0x00000040, 0x00004040,
414 0x9100, 0x07ffffff, 0x03000000,
415 0x9100, 0x07ffffff, 0x03000000,
416 0x8e88, 0x01ff1f3f, 0x00000000,
417 0x8e88, 0x01ff1f3f, 0x00000000,
418 0x8e88, 0x01ff1f3f, 0x00000000,
419 0x8e84, 0x01ff1f3f, 0x00000000,
420 0x8e84, 0x01ff1f3f, 0x00000000,
421 0x8e84, 0x01ff1f3f, 0x00000000,
422 0x9060, 0x0000007f, 0x00000020,
423 0x9508, 0x00010000, 0x00010000,
424 0xac14, 0x000003ff, 0x00000003,
425 0xac14, 0x000003ff, 0x00000003,
426 0xac14, 0x000003ff, 0x00000003,
427 0xac10, 0xffffffff, 0x00000000,
428 0xac10, 0xffffffff, 0x00000000,
429 0xac10, 0xffffffff, 0x00000000,
430 0xac0c, 0xffffffff, 0x00001032,
431 0xac0c, 0xffffffff, 0x00001032,
432 0xac0c, 0xffffffff, 0x00001032,
433 0x88d4, 0x0000001f, 0x00000010,
434 0x88d4, 0x0000001f, 0x00000010,
435 0x88d4, 0x0000001f, 0x00000010,
436 0x15c0, 0x000c0fc0, 0x000c0400
439 static const u32 oland_golden_rlc_registers[] =
441 0xc424, 0xffffffff, 0x00601005,
442 0xc47c, 0xffffffff, 0x10104040,
443 0xc488, 0xffffffff, 0x0100000a,
444 0xc314, 0xffffffff, 0x00000800,
445 0xc30c, 0xffffffff, 0x800000f4
448 static const u32 oland_golden_registers[] =
450 0x9a10, 0x00010000, 0x00018208,
451 0x9830, 0xffffffff, 0x00000000,
452 0x9834, 0xf00fffff, 0x00000400,
453 0x9838, 0x0002021c, 0x00020200,
454 0xc78, 0x00000080, 0x00000000,
455 0xd030, 0x000300c0, 0x00800040,
456 0xd830, 0x000300c0, 0x00800040,
457 0x5bb0, 0x000000f0, 0x00000070,
458 0x5bc0, 0x00200000, 0x50100000,
459 0x7030, 0x31000311, 0x00000011,
460 0x2ae4, 0x00073ffe, 0x000022a2,
461 0x240c, 0x000007ff, 0x00000000,
462 0x8a14, 0xf000001f, 0x00000007,
463 0x8b24, 0xffffffff, 0x00ffffff,
464 0x8b10, 0x0000ff0f, 0x00000000,
465 0x28a4c, 0x07ffffff, 0x4e000000,
466 0x28350, 0x3f3f3fff, 0x00000082,
467 0x30, 0x000000ff, 0x0040,
468 0x34, 0x00000040, 0x00004040,
469 0x9100, 0x07ffffff, 0x03000000,
470 0x9060, 0x0000007f, 0x00000020,
471 0x9508, 0x00010000, 0x00010000,
472 0xac14, 0x000003ff, 0x000000f3,
473 0xac10, 0xffffffff, 0x00000000,
474 0xac0c, 0xffffffff, 0x00003210,
475 0x88d4, 0x0000001f, 0x00000010,
476 0x15c0, 0x000c0fc0, 0x000c0400
479 static const u32 hainan_golden_registers[] =
481 0x9a10, 0x00010000, 0x00018208,
482 0x9830, 0xffffffff, 0x00000000,
483 0x9834, 0xf00fffff, 0x00000400,
484 0x9838, 0x0002021c, 0x00020200,
485 0xd0c0, 0xff000fff, 0x00000100,
486 0xd030, 0x000300c0, 0x00800040,
487 0xd8c0, 0xff000fff, 0x00000100,
488 0xd830, 0x000300c0, 0x00800040,
489 0x2ae4, 0x00073ffe, 0x000022a2,
490 0x240c, 0x000007ff, 0x00000000,
491 0x8a14, 0xf000001f, 0x00000007,
492 0x8b24, 0xffffffff, 0x00ffffff,
493 0x8b10, 0x0000ff0f, 0x00000000,
494 0x28a4c, 0x07ffffff, 0x4e000000,
495 0x28350, 0x3f3f3fff, 0x00000000,
496 0x30, 0x000000ff, 0x0040,
497 0x34, 0x00000040, 0x00004040,
498 0x9100, 0x03e00000, 0x03600000,
499 0x9060, 0x0000007f, 0x00000020,
500 0x9508, 0x00010000, 0x00010000,
501 0xac14, 0x000003ff, 0x000000f1,
502 0xac10, 0xffffffff, 0x00000000,
503 0xac0c, 0xffffffff, 0x00003210,
504 0x88d4, 0x0000001f, 0x00000010,
505 0x15c0, 0x000c0fc0, 0x000c0400
508 static const u32 hainan_golden_registers2[] =
510 0x98f8, 0xffffffff, 0x02010001
513 static const u32 tahiti_mgcg_cgcg_init[] =
515 0xc400, 0xffffffff, 0xfffffffc,
516 0x802c, 0xffffffff, 0xe0000000,
517 0x9a60, 0xffffffff, 0x00000100,
518 0x92a4, 0xffffffff, 0x00000100,
519 0xc164, 0xffffffff, 0x00000100,
520 0x9774, 0xffffffff, 0x00000100,
521 0x8984, 0xffffffff, 0x06000100,
522 0x8a18, 0xffffffff, 0x00000100,
523 0x92a0, 0xffffffff, 0x00000100,
524 0xc380, 0xffffffff, 0x00000100,
525 0x8b28, 0xffffffff, 0x00000100,
526 0x9144, 0xffffffff, 0x00000100,
527 0x8d88, 0xffffffff, 0x00000100,
528 0x8d8c, 0xffffffff, 0x00000100,
529 0x9030, 0xffffffff, 0x00000100,
530 0x9034, 0xffffffff, 0x00000100,
531 0x9038, 0xffffffff, 0x00000100,
532 0x903c, 0xffffffff, 0x00000100,
533 0xad80, 0xffffffff, 0x00000100,
534 0xac54, 0xffffffff, 0x00000100,
535 0x897c, 0xffffffff, 0x06000100,
536 0x9868, 0xffffffff, 0x00000100,
537 0x9510, 0xffffffff, 0x00000100,
538 0xaf04, 0xffffffff, 0x00000100,
539 0xae04, 0xffffffff, 0x00000100,
540 0x949c, 0xffffffff, 0x00000100,
541 0x802c, 0xffffffff, 0xe0000000,
542 0x9160, 0xffffffff, 0x00010000,
543 0x9164, 0xffffffff, 0x00030002,
544 0x9168, 0xffffffff, 0x00040007,
545 0x916c, 0xffffffff, 0x00060005,
546 0x9170, 0xffffffff, 0x00090008,
547 0x9174, 0xffffffff, 0x00020001,
548 0x9178, 0xffffffff, 0x00040003,
549 0x917c, 0xffffffff, 0x00000007,
550 0x9180, 0xffffffff, 0x00060005,
551 0x9184, 0xffffffff, 0x00090008,
552 0x9188, 0xffffffff, 0x00030002,
553 0x918c, 0xffffffff, 0x00050004,
554 0x9190, 0xffffffff, 0x00000008,
555 0x9194, 0xffffffff, 0x00070006,
556 0x9198, 0xffffffff, 0x000a0009,
557 0x919c, 0xffffffff, 0x00040003,
558 0x91a0, 0xffffffff, 0x00060005,
559 0x91a4, 0xffffffff, 0x00000009,
560 0x91a8, 0xffffffff, 0x00080007,
561 0x91ac, 0xffffffff, 0x000b000a,
562 0x91b0, 0xffffffff, 0x00050004,
563 0x91b4, 0xffffffff, 0x00070006,
564 0x91b8, 0xffffffff, 0x0008000b,
565 0x91bc, 0xffffffff, 0x000a0009,
566 0x91c0, 0xffffffff, 0x000d000c,
567 0x91c4, 0xffffffff, 0x00060005,
568 0x91c8, 0xffffffff, 0x00080007,
569 0x91cc, 0xffffffff, 0x0000000b,
570 0x91d0, 0xffffffff, 0x000a0009,
571 0x91d4, 0xffffffff, 0x000d000c,
572 0x91d8, 0xffffffff, 0x00070006,
573 0x91dc, 0xffffffff, 0x00090008,
574 0x91e0, 0xffffffff, 0x0000000c,
575 0x91e4, 0xffffffff, 0x000b000a,
576 0x91e8, 0xffffffff, 0x000e000d,
577 0x91ec, 0xffffffff, 0x00080007,
578 0x91f0, 0xffffffff, 0x000a0009,
579 0x91f4, 0xffffffff, 0x0000000d,
580 0x91f8, 0xffffffff, 0x000c000b,
581 0x91fc, 0xffffffff, 0x000f000e,
582 0x9200, 0xffffffff, 0x00090008,
583 0x9204, 0xffffffff, 0x000b000a,
584 0x9208, 0xffffffff, 0x000c000f,
585 0x920c, 0xffffffff, 0x000e000d,
586 0x9210, 0xffffffff, 0x00110010,
587 0x9214, 0xffffffff, 0x000a0009,
588 0x9218, 0xffffffff, 0x000c000b,
589 0x921c, 0xffffffff, 0x0000000f,
590 0x9220, 0xffffffff, 0x000e000d,
591 0x9224, 0xffffffff, 0x00110010,
592 0x9228, 0xffffffff, 0x000b000a,
593 0x922c, 0xffffffff, 0x000d000c,
594 0x9230, 0xffffffff, 0x00000010,
595 0x9234, 0xffffffff, 0x000f000e,
596 0x9238, 0xffffffff, 0x00120011,
597 0x923c, 0xffffffff, 0x000c000b,
598 0x9240, 0xffffffff, 0x000e000d,
599 0x9244, 0xffffffff, 0x00000011,
600 0x9248, 0xffffffff, 0x0010000f,
601 0x924c, 0xffffffff, 0x00130012,
602 0x9250, 0xffffffff, 0x000d000c,
603 0x9254, 0xffffffff, 0x000f000e,
604 0x9258, 0xffffffff, 0x00100013,
605 0x925c, 0xffffffff, 0x00120011,
606 0x9260, 0xffffffff, 0x00150014,
607 0x9264, 0xffffffff, 0x000e000d,
608 0x9268, 0xffffffff, 0x0010000f,
609 0x926c, 0xffffffff, 0x00000013,
610 0x9270, 0xffffffff, 0x00120011,
611 0x9274, 0xffffffff, 0x00150014,
612 0x9278, 0xffffffff, 0x000f000e,
613 0x927c, 0xffffffff, 0x00110010,
614 0x9280, 0xffffffff, 0x00000014,
615 0x9284, 0xffffffff, 0x00130012,
616 0x9288, 0xffffffff, 0x00160015,
617 0x928c, 0xffffffff, 0x0010000f,
618 0x9290, 0xffffffff, 0x00120011,
619 0x9294, 0xffffffff, 0x00000015,
620 0x9298, 0xffffffff, 0x00140013,
621 0x929c, 0xffffffff, 0x00170016,
622 0x9150, 0xffffffff, 0x96940200,
623 0x8708, 0xffffffff, 0x00900100,
624 0xc478, 0xffffffff, 0x00000080,
625 0xc404, 0xffffffff, 0x0020003f,
626 0x30, 0xffffffff, 0x0000001c,
627 0x34, 0x000f0000, 0x000f0000,
628 0x160c, 0xffffffff, 0x00000100,
629 0x1024, 0xffffffff, 0x00000100,
630 0x102c, 0x00000101, 0x00000000,
631 0x20a8, 0xffffffff, 0x00000104,
632 0x264c, 0x000c0000, 0x000c0000,
633 0x2648, 0x000c0000, 0x000c0000,
634 0x55e4, 0xff000fff, 0x00000100,
635 0x55e8, 0x00000001, 0x00000001,
636 0x2f50, 0x00000001, 0x00000001,
637 0x30cc, 0xc0000fff, 0x00000104,
638 0xc1e4, 0x00000001, 0x00000001,
639 0xd0c0, 0xfffffff0, 0x00000100,
640 0xd8c0, 0xfffffff0, 0x00000100
643 static const u32 pitcairn_mgcg_cgcg_init[] =
645 0xc400, 0xffffffff, 0xfffffffc,
646 0x802c, 0xffffffff, 0xe0000000,
647 0x9a60, 0xffffffff, 0x00000100,
648 0x92a4, 0xffffffff, 0x00000100,
649 0xc164, 0xffffffff, 0x00000100,
650 0x9774, 0xffffffff, 0x00000100,
651 0x8984, 0xffffffff, 0x06000100,
652 0x8a18, 0xffffffff, 0x00000100,
653 0x92a0, 0xffffffff, 0x00000100,
654 0xc380, 0xffffffff, 0x00000100,
655 0x8b28, 0xffffffff, 0x00000100,
656 0x9144, 0xffffffff, 0x00000100,
657 0x8d88, 0xffffffff, 0x00000100,
658 0x8d8c, 0xffffffff, 0x00000100,
659 0x9030, 0xffffffff, 0x00000100,
660 0x9034, 0xffffffff, 0x00000100,
661 0x9038, 0xffffffff, 0x00000100,
662 0x903c, 0xffffffff, 0x00000100,
663 0xad80, 0xffffffff, 0x00000100,
664 0xac54, 0xffffffff, 0x00000100,
665 0x897c, 0xffffffff, 0x06000100,
666 0x9868, 0xffffffff, 0x00000100,
667 0x9510, 0xffffffff, 0x00000100,
668 0xaf04, 0xffffffff, 0x00000100,
669 0xae04, 0xffffffff, 0x00000100,
670 0x949c, 0xffffffff, 0x00000100,
671 0x802c, 0xffffffff, 0xe0000000,
672 0x9160, 0xffffffff, 0x00010000,
673 0x9164, 0xffffffff, 0x00030002,
674 0x9168, 0xffffffff, 0x00040007,
675 0x916c, 0xffffffff, 0x00060005,
676 0x9170, 0xffffffff, 0x00090008,
677 0x9174, 0xffffffff, 0x00020001,
678 0x9178, 0xffffffff, 0x00040003,
679 0x917c, 0xffffffff, 0x00000007,
680 0x9180, 0xffffffff, 0x00060005,
681 0x9184, 0xffffffff, 0x00090008,
682 0x9188, 0xffffffff, 0x00030002,
683 0x918c, 0xffffffff, 0x00050004,
684 0x9190, 0xffffffff, 0x00000008,
685 0x9194, 0xffffffff, 0x00070006,
686 0x9198, 0xffffffff, 0x000a0009,
687 0x919c, 0xffffffff, 0x00040003,
688 0x91a0, 0xffffffff, 0x00060005,
689 0x91a4, 0xffffffff, 0x00000009,
690 0x91a8, 0xffffffff, 0x00080007,
691 0x91ac, 0xffffffff, 0x000b000a,
692 0x91b0, 0xffffffff, 0x00050004,
693 0x91b4, 0xffffffff, 0x00070006,
694 0x91b8, 0xffffffff, 0x0008000b,
695 0x91bc, 0xffffffff, 0x000a0009,
696 0x91c0, 0xffffffff, 0x000d000c,
697 0x9200, 0xffffffff, 0x00090008,
698 0x9204, 0xffffffff, 0x000b000a,
699 0x9208, 0xffffffff, 0x000c000f,
700 0x920c, 0xffffffff, 0x000e000d,
701 0x9210, 0xffffffff, 0x00110010,
702 0x9214, 0xffffffff, 0x000a0009,
703 0x9218, 0xffffffff, 0x000c000b,
704 0x921c, 0xffffffff, 0x0000000f,
705 0x9220, 0xffffffff, 0x000e000d,
706 0x9224, 0xffffffff, 0x00110010,
707 0x9228, 0xffffffff, 0x000b000a,
708 0x922c, 0xffffffff, 0x000d000c,
709 0x9230, 0xffffffff, 0x00000010,
710 0x9234, 0xffffffff, 0x000f000e,
711 0x9238, 0xffffffff, 0x00120011,
712 0x923c, 0xffffffff, 0x000c000b,
713 0x9240, 0xffffffff, 0x000e000d,
714 0x9244, 0xffffffff, 0x00000011,
715 0x9248, 0xffffffff, 0x0010000f,
716 0x924c, 0xffffffff, 0x00130012,
717 0x9250, 0xffffffff, 0x000d000c,
718 0x9254, 0xffffffff, 0x000f000e,
719 0x9258, 0xffffffff, 0x00100013,
720 0x925c, 0xffffffff, 0x00120011,
721 0x9260, 0xffffffff, 0x00150014,
722 0x9150, 0xffffffff, 0x96940200,
723 0x8708, 0xffffffff, 0x00900100,
724 0xc478, 0xffffffff, 0x00000080,
725 0xc404, 0xffffffff, 0x0020003f,
726 0x30, 0xffffffff, 0x0000001c,
727 0x34, 0x000f0000, 0x000f0000,
728 0x160c, 0xffffffff, 0x00000100,
729 0x1024, 0xffffffff, 0x00000100,
730 0x102c, 0x00000101, 0x00000000,
731 0x20a8, 0xffffffff, 0x00000104,
732 0x55e4, 0xff000fff, 0x00000100,
733 0x55e8, 0x00000001, 0x00000001,
734 0x2f50, 0x00000001, 0x00000001,
735 0x30cc, 0xc0000fff, 0x00000104,
736 0xc1e4, 0x00000001, 0x00000001,
737 0xd0c0, 0xfffffff0, 0x00000100,
738 0xd8c0, 0xfffffff0, 0x00000100
741 static const u32 verde_mgcg_cgcg_init[] =
743 0xc400, 0xffffffff, 0xfffffffc,
744 0x802c, 0xffffffff, 0xe0000000,
745 0x9a60, 0xffffffff, 0x00000100,
746 0x92a4, 0xffffffff, 0x00000100,
747 0xc164, 0xffffffff, 0x00000100,
748 0x9774, 0xffffffff, 0x00000100,
749 0x8984, 0xffffffff, 0x06000100,
750 0x8a18, 0xffffffff, 0x00000100,
751 0x92a0, 0xffffffff, 0x00000100,
752 0xc380, 0xffffffff, 0x00000100,
753 0x8b28, 0xffffffff, 0x00000100,
754 0x9144, 0xffffffff, 0x00000100,
755 0x8d88, 0xffffffff, 0x00000100,
756 0x8d8c, 0xffffffff, 0x00000100,
757 0x9030, 0xffffffff, 0x00000100,
758 0x9034, 0xffffffff, 0x00000100,
759 0x9038, 0xffffffff, 0x00000100,
760 0x903c, 0xffffffff, 0x00000100,
761 0xad80, 0xffffffff, 0x00000100,
762 0xac54, 0xffffffff, 0x00000100,
763 0x897c, 0xffffffff, 0x06000100,
764 0x9868, 0xffffffff, 0x00000100,
765 0x9510, 0xffffffff, 0x00000100,
766 0xaf04, 0xffffffff, 0x00000100,
767 0xae04, 0xffffffff, 0x00000100,
768 0x949c, 0xffffffff, 0x00000100,
769 0x802c, 0xffffffff, 0xe0000000,
770 0x9160, 0xffffffff, 0x00010000,
771 0x9164, 0xffffffff, 0x00030002,
772 0x9168, 0xffffffff, 0x00040007,
773 0x916c, 0xffffffff, 0x00060005,
774 0x9170, 0xffffffff, 0x00090008,
775 0x9174, 0xffffffff, 0x00020001,
776 0x9178, 0xffffffff, 0x00040003,
777 0x917c, 0xffffffff, 0x00000007,
778 0x9180, 0xffffffff, 0x00060005,
779 0x9184, 0xffffffff, 0x00090008,
780 0x9188, 0xffffffff, 0x00030002,
781 0x918c, 0xffffffff, 0x00050004,
782 0x9190, 0xffffffff, 0x00000008,
783 0x9194, 0xffffffff, 0x00070006,
784 0x9198, 0xffffffff, 0x000a0009,
785 0x919c, 0xffffffff, 0x00040003,
786 0x91a0, 0xffffffff, 0x00060005,
787 0x91a4, 0xffffffff, 0x00000009,
788 0x91a8, 0xffffffff, 0x00080007,
789 0x91ac, 0xffffffff, 0x000b000a,
790 0x91b0, 0xffffffff, 0x00050004,
791 0x91b4, 0xffffffff, 0x00070006,
792 0x91b8, 0xffffffff, 0x0008000b,
793 0x91bc, 0xffffffff, 0x000a0009,
794 0x91c0, 0xffffffff, 0x000d000c,
795 0x9200, 0xffffffff, 0x00090008,
796 0x9204, 0xffffffff, 0x000b000a,
797 0x9208, 0xffffffff, 0x000c000f,
798 0x920c, 0xffffffff, 0x000e000d,
799 0x9210, 0xffffffff, 0x00110010,
800 0x9214, 0xffffffff, 0x000a0009,
801 0x9218, 0xffffffff, 0x000c000b,
802 0x921c, 0xffffffff, 0x0000000f,
803 0x9220, 0xffffffff, 0x000e000d,
804 0x9224, 0xffffffff, 0x00110010,
805 0x9228, 0xffffffff, 0x000b000a,
806 0x922c, 0xffffffff, 0x000d000c,
807 0x9230, 0xffffffff, 0x00000010,
808 0x9234, 0xffffffff, 0x000f000e,
809 0x9238, 0xffffffff, 0x00120011,
810 0x923c, 0xffffffff, 0x000c000b,
811 0x9240, 0xffffffff, 0x000e000d,
812 0x9244, 0xffffffff, 0x00000011,
813 0x9248, 0xffffffff, 0x0010000f,
814 0x924c, 0xffffffff, 0x00130012,
815 0x9250, 0xffffffff, 0x000d000c,
816 0x9254, 0xffffffff, 0x000f000e,
817 0x9258, 0xffffffff, 0x00100013,
818 0x925c, 0xffffffff, 0x00120011,
819 0x9260, 0xffffffff, 0x00150014,
820 0x9150, 0xffffffff, 0x96940200,
821 0x8708, 0xffffffff, 0x00900100,
822 0xc478, 0xffffffff, 0x00000080,
823 0xc404, 0xffffffff, 0x0020003f,
824 0x30, 0xffffffff, 0x0000001c,
825 0x34, 0x000f0000, 0x000f0000,
826 0x160c, 0xffffffff, 0x00000100,
827 0x1024, 0xffffffff, 0x00000100,
828 0x102c, 0x00000101, 0x00000000,
829 0x20a8, 0xffffffff, 0x00000104,
830 0x264c, 0x000c0000, 0x000c0000,
831 0x2648, 0x000c0000, 0x000c0000,
832 0x55e4, 0xff000fff, 0x00000100,
833 0x55e8, 0x00000001, 0x00000001,
834 0x2f50, 0x00000001, 0x00000001,
835 0x30cc, 0xc0000fff, 0x00000104,
836 0xc1e4, 0x00000001, 0x00000001,
837 0xd0c0, 0xfffffff0, 0x00000100,
838 0xd8c0, 0xfffffff0, 0x00000100
841 static const u32 oland_mgcg_cgcg_init[] =
843 0xc400, 0xffffffff, 0xfffffffc,
844 0x802c, 0xffffffff, 0xe0000000,
845 0x9a60, 0xffffffff, 0x00000100,
846 0x92a4, 0xffffffff, 0x00000100,
847 0xc164, 0xffffffff, 0x00000100,
848 0x9774, 0xffffffff, 0x00000100,
849 0x8984, 0xffffffff, 0x06000100,
850 0x8a18, 0xffffffff, 0x00000100,
851 0x92a0, 0xffffffff, 0x00000100,
852 0xc380, 0xffffffff, 0x00000100,
853 0x8b28, 0xffffffff, 0x00000100,
854 0x9144, 0xffffffff, 0x00000100,
855 0x8d88, 0xffffffff, 0x00000100,
856 0x8d8c, 0xffffffff, 0x00000100,
857 0x9030, 0xffffffff, 0x00000100,
858 0x9034, 0xffffffff, 0x00000100,
859 0x9038, 0xffffffff, 0x00000100,
860 0x903c, 0xffffffff, 0x00000100,
861 0xad80, 0xffffffff, 0x00000100,
862 0xac54, 0xffffffff, 0x00000100,
863 0x897c, 0xffffffff, 0x06000100,
864 0x9868, 0xffffffff, 0x00000100,
865 0x9510, 0xffffffff, 0x00000100,
866 0xaf04, 0xffffffff, 0x00000100,
867 0xae04, 0xffffffff, 0x00000100,
868 0x949c, 0xffffffff, 0x00000100,
869 0x802c, 0xffffffff, 0xe0000000,
870 0x9160, 0xffffffff, 0x00010000,
871 0x9164, 0xffffffff, 0x00030002,
872 0x9168, 0xffffffff, 0x00040007,
873 0x916c, 0xffffffff, 0x00060005,
874 0x9170, 0xffffffff, 0x00090008,
875 0x9174, 0xffffffff, 0x00020001,
876 0x9178, 0xffffffff, 0x00040003,
877 0x917c, 0xffffffff, 0x00000007,
878 0x9180, 0xffffffff, 0x00060005,
879 0x9184, 0xffffffff, 0x00090008,
880 0x9188, 0xffffffff, 0x00030002,
881 0x918c, 0xffffffff, 0x00050004,
882 0x9190, 0xffffffff, 0x00000008,
883 0x9194, 0xffffffff, 0x00070006,
884 0x9198, 0xffffffff, 0x000a0009,
885 0x919c, 0xffffffff, 0x00040003,
886 0x91a0, 0xffffffff, 0x00060005,
887 0x91a4, 0xffffffff, 0x00000009,
888 0x91a8, 0xffffffff, 0x00080007,
889 0x91ac, 0xffffffff, 0x000b000a,
890 0x91b0, 0xffffffff, 0x00050004,
891 0x91b4, 0xffffffff, 0x00070006,
892 0x91b8, 0xffffffff, 0x0008000b,
893 0x91bc, 0xffffffff, 0x000a0009,
894 0x91c0, 0xffffffff, 0x000d000c,
895 0x91c4, 0xffffffff, 0x00060005,
896 0x91c8, 0xffffffff, 0x00080007,
897 0x91cc, 0xffffffff, 0x0000000b,
898 0x91d0, 0xffffffff, 0x000a0009,
899 0x91d4, 0xffffffff, 0x000d000c,
900 0x9150, 0xffffffff, 0x96940200,
901 0x8708, 0xffffffff, 0x00900100,
902 0xc478, 0xffffffff, 0x00000080,
903 0xc404, 0xffffffff, 0x0020003f,
904 0x30, 0xffffffff, 0x0000001c,
905 0x34, 0x000f0000, 0x000f0000,
906 0x160c, 0xffffffff, 0x00000100,
907 0x1024, 0xffffffff, 0x00000100,
908 0x102c, 0x00000101, 0x00000000,
909 0x20a8, 0xffffffff, 0x00000104,
910 0x264c, 0x000c0000, 0x000c0000,
911 0x2648, 0x000c0000, 0x000c0000,
912 0x55e4, 0xff000fff, 0x00000100,
913 0x55e8, 0x00000001, 0x00000001,
914 0x2f50, 0x00000001, 0x00000001,
915 0x30cc, 0xc0000fff, 0x00000104,
916 0xc1e4, 0x00000001, 0x00000001,
917 0xd0c0, 0xfffffff0, 0x00000100,
918 0xd8c0, 0xfffffff0, 0x00000100
921 static const u32 hainan_mgcg_cgcg_init[] =
923 0xc400, 0xffffffff, 0xfffffffc,
924 0x802c, 0xffffffff, 0xe0000000,
925 0x9a60, 0xffffffff, 0x00000100,
926 0x92a4, 0xffffffff, 0x00000100,
927 0xc164, 0xffffffff, 0x00000100,
928 0x9774, 0xffffffff, 0x00000100,
929 0x8984, 0xffffffff, 0x06000100,
930 0x8a18, 0xffffffff, 0x00000100,
931 0x92a0, 0xffffffff, 0x00000100,
932 0xc380, 0xffffffff, 0x00000100,
933 0x8b28, 0xffffffff, 0x00000100,
934 0x9144, 0xffffffff, 0x00000100,
935 0x8d88, 0xffffffff, 0x00000100,
936 0x8d8c, 0xffffffff, 0x00000100,
937 0x9030, 0xffffffff, 0x00000100,
938 0x9034, 0xffffffff, 0x00000100,
939 0x9038, 0xffffffff, 0x00000100,
940 0x903c, 0xffffffff, 0x00000100,
941 0xad80, 0xffffffff, 0x00000100,
942 0xac54, 0xffffffff, 0x00000100,
943 0x897c, 0xffffffff, 0x06000100,
944 0x9868, 0xffffffff, 0x00000100,
945 0x9510, 0xffffffff, 0x00000100,
946 0xaf04, 0xffffffff, 0x00000100,
947 0xae04, 0xffffffff, 0x00000100,
948 0x949c, 0xffffffff, 0x00000100,
949 0x802c, 0xffffffff, 0xe0000000,
950 0x9160, 0xffffffff, 0x00010000,
951 0x9164, 0xffffffff, 0x00030002,
952 0x9168, 0xffffffff, 0x00040007,
953 0x916c, 0xffffffff, 0x00060005,
954 0x9170, 0xffffffff, 0x00090008,
955 0x9174, 0xffffffff, 0x00020001,
956 0x9178, 0xffffffff, 0x00040003,
957 0x917c, 0xffffffff, 0x00000007,
958 0x9180, 0xffffffff, 0x00060005,
959 0x9184, 0xffffffff, 0x00090008,
960 0x9188, 0xffffffff, 0x00030002,
961 0x918c, 0xffffffff, 0x00050004,
962 0x9190, 0xffffffff, 0x00000008,
963 0x9194, 0xffffffff, 0x00070006,
964 0x9198, 0xffffffff, 0x000a0009,
965 0x919c, 0xffffffff, 0x00040003,
966 0x91a0, 0xffffffff, 0x00060005,
967 0x91a4, 0xffffffff, 0x00000009,
968 0x91a8, 0xffffffff, 0x00080007,
969 0x91ac, 0xffffffff, 0x000b000a,
970 0x91b0, 0xffffffff, 0x00050004,
971 0x91b4, 0xffffffff, 0x00070006,
972 0x91b8, 0xffffffff, 0x0008000b,
973 0x91bc, 0xffffffff, 0x000a0009,
974 0x91c0, 0xffffffff, 0x000d000c,
975 0x91c4, 0xffffffff, 0x00060005,
976 0x91c8, 0xffffffff, 0x00080007,
977 0x91cc, 0xffffffff, 0x0000000b,
978 0x91d0, 0xffffffff, 0x000a0009,
979 0x91d4, 0xffffffff, 0x000d000c,
980 0x9150, 0xffffffff, 0x96940200,
981 0x8708, 0xffffffff, 0x00900100,
982 0xc478, 0xffffffff, 0x00000080,
983 0xc404, 0xffffffff, 0x0020003f,
984 0x30, 0xffffffff, 0x0000001c,
985 0x34, 0x000f0000, 0x000f0000,
986 0x160c, 0xffffffff, 0x00000100,
987 0x1024, 0xffffffff, 0x00000100,
988 0x20a8, 0xffffffff, 0x00000104,
989 0x264c, 0x000c0000, 0x000c0000,
990 0x2648, 0x000c0000, 0x000c0000,
991 0x2f50, 0x00000001, 0x00000001,
992 0x30cc, 0xc0000fff, 0x00000104,
993 0xc1e4, 0x00000001, 0x00000001,
994 0xd0c0, 0xfffffff0, 0x00000100,
995 0xd8c0, 0xfffffff0, 0x00000100
998 static u32 verde_pg_init[] =
1000 0x353c, 0xffffffff, 0x40000,
1001 0x3538, 0xffffffff, 0x200010ff,
1002 0x353c, 0xffffffff, 0x0,
1003 0x353c, 0xffffffff, 0x0,
1004 0x353c, 0xffffffff, 0x0,
1005 0x353c, 0xffffffff, 0x0,
1006 0x353c, 0xffffffff, 0x0,
1007 0x353c, 0xffffffff, 0x7007,
1008 0x3538, 0xffffffff, 0x300010ff,
1009 0x353c, 0xffffffff, 0x0,
1010 0x353c, 0xffffffff, 0x0,
1011 0x353c, 0xffffffff, 0x0,
1012 0x353c, 0xffffffff, 0x0,
1013 0x353c, 0xffffffff, 0x0,
1014 0x353c, 0xffffffff, 0x400000,
1015 0x3538, 0xffffffff, 0x100010ff,
1016 0x353c, 0xffffffff, 0x0,
1017 0x353c, 0xffffffff, 0x0,
1018 0x353c, 0xffffffff, 0x0,
1019 0x353c, 0xffffffff, 0x0,
1020 0x353c, 0xffffffff, 0x0,
1021 0x353c, 0xffffffff, 0x120200,
1022 0x3538, 0xffffffff, 0x500010ff,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x0,
1027 0x353c, 0xffffffff, 0x0,
1028 0x353c, 0xffffffff, 0x1e1e16,
1029 0x3538, 0xffffffff, 0x600010ff,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x171f1e,
1036 0x3538, 0xffffffff, 0x700010ff,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x3538, 0xffffffff, 0x9ff,
1044 0x3500, 0xffffffff, 0x0,
1045 0x3504, 0xffffffff, 0x10000800,
1046 0x3504, 0xffffffff, 0xf,
1047 0x3504, 0xffffffff, 0xf,
1048 0x3500, 0xffffffff, 0x4,
1049 0x3504, 0xffffffff, 0x1000051e,
1050 0x3504, 0xffffffff, 0xffff,
1051 0x3504, 0xffffffff, 0xffff,
1052 0x3500, 0xffffffff, 0x8,
1053 0x3504, 0xffffffff, 0x80500,
1054 0x3500, 0xffffffff, 0x12,
1055 0x3504, 0xffffffff, 0x9050c,
1056 0x3500, 0xffffffff, 0x1d,
1057 0x3504, 0xffffffff, 0xb052c,
1058 0x3500, 0xffffffff, 0x2a,
1059 0x3504, 0xffffffff, 0x1053e,
1060 0x3500, 0xffffffff, 0x2d,
1061 0x3504, 0xffffffff, 0x10546,
1062 0x3500, 0xffffffff, 0x30,
1063 0x3504, 0xffffffff, 0xa054e,
1064 0x3500, 0xffffffff, 0x3c,
1065 0x3504, 0xffffffff, 0x1055f,
1066 0x3500, 0xffffffff, 0x3f,
1067 0x3504, 0xffffffff, 0x10567,
1068 0x3500, 0xffffffff, 0x42,
1069 0x3504, 0xffffffff, 0x1056f,
1070 0x3500, 0xffffffff, 0x45,
1071 0x3504, 0xffffffff, 0x10572,
1072 0x3500, 0xffffffff, 0x48,
1073 0x3504, 0xffffffff, 0x20575,
1074 0x3500, 0xffffffff, 0x4c,
1075 0x3504, 0xffffffff, 0x190801,
1076 0x3500, 0xffffffff, 0x67,
1077 0x3504, 0xffffffff, 0x1082a,
1078 0x3500, 0xffffffff, 0x6a,
1079 0x3504, 0xffffffff, 0x1b082d,
1080 0x3500, 0xffffffff, 0x87,
1081 0x3504, 0xffffffff, 0x310851,
1082 0x3500, 0xffffffff, 0xba,
1083 0x3504, 0xffffffff, 0x891,
1084 0x3500, 0xffffffff, 0xbc,
1085 0x3504, 0xffffffff, 0x893,
1086 0x3500, 0xffffffff, 0xbe,
1087 0x3504, 0xffffffff, 0x20895,
1088 0x3500, 0xffffffff, 0xc2,
1089 0x3504, 0xffffffff, 0x20899,
1090 0x3500, 0xffffffff, 0xc6,
1091 0x3504, 0xffffffff, 0x2089d,
1092 0x3500, 0xffffffff, 0xca,
1093 0x3504, 0xffffffff, 0x8a1,
1094 0x3500, 0xffffffff, 0xcc,
1095 0x3504, 0xffffffff, 0x8a3,
1096 0x3500, 0xffffffff, 0xce,
1097 0x3504, 0xffffffff, 0x308a5,
1098 0x3500, 0xffffffff, 0xd3,
1099 0x3504, 0xffffffff, 0x6d08cd,
1100 0x3500, 0xffffffff, 0x142,
1101 0x3504, 0xffffffff, 0x2000095a,
1102 0x3504, 0xffffffff, 0x1,
1103 0x3500, 0xffffffff, 0x144,
1104 0x3504, 0xffffffff, 0x301f095b,
1105 0x3500, 0xffffffff, 0x165,
1106 0x3504, 0xffffffff, 0xc094d,
1107 0x3500, 0xffffffff, 0x173,
1108 0x3504, 0xffffffff, 0xf096d,
1109 0x3500, 0xffffffff, 0x184,
1110 0x3504, 0xffffffff, 0x15097f,
1111 0x3500, 0xffffffff, 0x19b,
1112 0x3504, 0xffffffff, 0xc0998,
1113 0x3500, 0xffffffff, 0x1a9,
1114 0x3504, 0xffffffff, 0x409a7,
1115 0x3500, 0xffffffff, 0x1af,
1116 0x3504, 0xffffffff, 0xcdc,
1117 0x3500, 0xffffffff, 0x1b1,
1118 0x3504, 0xffffffff, 0x800,
1119 0x3508, 0xffffffff, 0x6c9b2000,
1120 0x3510, 0xfc00, 0x2000,
1121 0x3544, 0xffffffff, 0xfc0,
1122 0x28d4, 0x00000100, 0x100
1125 static void si_init_golden_registers(struct radeon_device *rdev)
1127 switch (rdev->family) {
1129 radeon_program_register_sequence(rdev,
1130 tahiti_golden_registers,
1131 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1132 radeon_program_register_sequence(rdev,
1133 tahiti_golden_rlc_registers,
1134 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1135 radeon_program_register_sequence(rdev,
1136 tahiti_mgcg_cgcg_init,
1137 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1138 radeon_program_register_sequence(rdev,
1139 tahiti_golden_registers2,
1140 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1143 radeon_program_register_sequence(rdev,
1144 pitcairn_golden_registers,
1145 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1146 radeon_program_register_sequence(rdev,
1147 pitcairn_golden_rlc_registers,
1148 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1149 radeon_program_register_sequence(rdev,
1150 pitcairn_mgcg_cgcg_init,
1151 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1154 radeon_program_register_sequence(rdev,
1155 verde_golden_registers,
1156 (const u32)ARRAY_SIZE(verde_golden_registers));
1157 radeon_program_register_sequence(rdev,
1158 verde_golden_rlc_registers,
1159 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1160 radeon_program_register_sequence(rdev,
1161 verde_mgcg_cgcg_init,
1162 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1163 radeon_program_register_sequence(rdev,
1165 (const u32)ARRAY_SIZE(verde_pg_init));
1168 radeon_program_register_sequence(rdev,
1169 oland_golden_registers,
1170 (const u32)ARRAY_SIZE(oland_golden_registers));
1171 radeon_program_register_sequence(rdev,
1172 oland_golden_rlc_registers,
1173 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1174 radeon_program_register_sequence(rdev,
1175 oland_mgcg_cgcg_init,
1176 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1179 radeon_program_register_sequence(rdev,
1180 hainan_golden_registers,
1181 (const u32)ARRAY_SIZE(hainan_golden_registers));
1182 radeon_program_register_sequence(rdev,
1183 hainan_golden_registers2,
1184 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1185 radeon_program_register_sequence(rdev,
1186 hainan_mgcg_cgcg_init,
1187 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1195 * si_get_allowed_info_register - fetch the register for the info ioctl
1197 * @rdev: radeon_device pointer
1198 * @reg: register offset in bytes
1199 * @val: register value
1201 * Returns 0 for success or -EINVAL for an invalid register
1204 int si_get_allowed_info_register(struct radeon_device *rdev,
1210 case GRBM_STATUS_SE0:
1211 case GRBM_STATUS_SE1:
1214 case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1215 case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1224 #define PCIE_BUS_CLK 10000
1225 #define TCLK (PCIE_BUS_CLK / 10)
1228 * si_get_xclk - get the xclk
1230 * @rdev: radeon_device pointer
1232 * Returns the reference clock used by the gfx engine
1235 u32 si_get_xclk(struct radeon_device *rdev)
1237 u32 reference_clock = rdev->clock.spll.reference_freq;
1240 tmp = RREG32(CG_CLKPIN_CNTL_2);
1241 if (tmp & MUX_TCLK_TO_XCLK)
1244 tmp = RREG32(CG_CLKPIN_CNTL);
1245 if (tmp & XTALIN_DIVIDE)
1246 return reference_clock / 4;
1248 return reference_clock;
1251 /* get temperature in millidegrees */
1252 int si_get_temp(struct radeon_device *rdev)
1255 int actual_temp = 0;
1257 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1263 actual_temp = temp & 0x1ff;
1265 actual_temp = (actual_temp * 1000);
1270 #define TAHITI_IO_MC_REGS_SIZE 36
1272 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1273 {0x0000006f, 0x03044000},
1274 {0x00000070, 0x0480c018},
1275 {0x00000071, 0x00000040},
1276 {0x00000072, 0x01000000},
1277 {0x00000074, 0x000000ff},
1278 {0x00000075, 0x00143400},
1279 {0x00000076, 0x08ec0800},
1280 {0x00000077, 0x040000cc},
1281 {0x00000079, 0x00000000},
1282 {0x0000007a, 0x21000409},
1283 {0x0000007c, 0x00000000},
1284 {0x0000007d, 0xe8000000},
1285 {0x0000007e, 0x044408a8},
1286 {0x0000007f, 0x00000003},
1287 {0x00000080, 0x00000000},
1288 {0x00000081, 0x01000000},
1289 {0x00000082, 0x02000000},
1290 {0x00000083, 0x00000000},
1291 {0x00000084, 0xe3f3e4f4},
1292 {0x00000085, 0x00052024},
1293 {0x00000087, 0x00000000},
1294 {0x00000088, 0x66036603},
1295 {0x00000089, 0x01000000},
1296 {0x0000008b, 0x1c0a0000},
1297 {0x0000008c, 0xff010000},
1298 {0x0000008e, 0xffffefff},
1299 {0x0000008f, 0xfff3efff},
1300 {0x00000090, 0xfff3efbf},
1301 {0x00000094, 0x00101101},
1302 {0x00000095, 0x00000fff},
1303 {0x00000096, 0x00116fff},
1304 {0x00000097, 0x60010000},
1305 {0x00000098, 0x10010000},
1306 {0x00000099, 0x00006000},
1307 {0x0000009a, 0x00001000},
1308 {0x0000009f, 0x00a77400}
1311 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1312 {0x0000006f, 0x03044000},
1313 {0x00000070, 0x0480c018},
1314 {0x00000071, 0x00000040},
1315 {0x00000072, 0x01000000},
1316 {0x00000074, 0x000000ff},
1317 {0x00000075, 0x00143400},
1318 {0x00000076, 0x08ec0800},
1319 {0x00000077, 0x040000cc},
1320 {0x00000079, 0x00000000},
1321 {0x0000007a, 0x21000409},
1322 {0x0000007c, 0x00000000},
1323 {0x0000007d, 0xe8000000},
1324 {0x0000007e, 0x044408a8},
1325 {0x0000007f, 0x00000003},
1326 {0x00000080, 0x00000000},
1327 {0x00000081, 0x01000000},
1328 {0x00000082, 0x02000000},
1329 {0x00000083, 0x00000000},
1330 {0x00000084, 0xe3f3e4f4},
1331 {0x00000085, 0x00052024},
1332 {0x00000087, 0x00000000},
1333 {0x00000088, 0x66036603},
1334 {0x00000089, 0x01000000},
1335 {0x0000008b, 0x1c0a0000},
1336 {0x0000008c, 0xff010000},
1337 {0x0000008e, 0xffffefff},
1338 {0x0000008f, 0xfff3efff},
1339 {0x00000090, 0xfff3efbf},
1340 {0x00000094, 0x00101101},
1341 {0x00000095, 0x00000fff},
1342 {0x00000096, 0x00116fff},
1343 {0x00000097, 0x60010000},
1344 {0x00000098, 0x10010000},
1345 {0x00000099, 0x00006000},
1346 {0x0000009a, 0x00001000},
1347 {0x0000009f, 0x00a47400}
1350 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1351 {0x0000006f, 0x03044000},
1352 {0x00000070, 0x0480c018},
1353 {0x00000071, 0x00000040},
1354 {0x00000072, 0x01000000},
1355 {0x00000074, 0x000000ff},
1356 {0x00000075, 0x00143400},
1357 {0x00000076, 0x08ec0800},
1358 {0x00000077, 0x040000cc},
1359 {0x00000079, 0x00000000},
1360 {0x0000007a, 0x21000409},
1361 {0x0000007c, 0x00000000},
1362 {0x0000007d, 0xe8000000},
1363 {0x0000007e, 0x044408a8},
1364 {0x0000007f, 0x00000003},
1365 {0x00000080, 0x00000000},
1366 {0x00000081, 0x01000000},
1367 {0x00000082, 0x02000000},
1368 {0x00000083, 0x00000000},
1369 {0x00000084, 0xe3f3e4f4},
1370 {0x00000085, 0x00052024},
1371 {0x00000087, 0x00000000},
1372 {0x00000088, 0x66036603},
1373 {0x00000089, 0x01000000},
1374 {0x0000008b, 0x1c0a0000},
1375 {0x0000008c, 0xff010000},
1376 {0x0000008e, 0xffffefff},
1377 {0x0000008f, 0xfff3efff},
1378 {0x00000090, 0xfff3efbf},
1379 {0x00000094, 0x00101101},
1380 {0x00000095, 0x00000fff},
1381 {0x00000096, 0x00116fff},
1382 {0x00000097, 0x60010000},
1383 {0x00000098, 0x10010000},
1384 {0x00000099, 0x00006000},
1385 {0x0000009a, 0x00001000},
1386 {0x0000009f, 0x00a37400}
1389 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1390 {0x0000006f, 0x03044000},
1391 {0x00000070, 0x0480c018},
1392 {0x00000071, 0x00000040},
1393 {0x00000072, 0x01000000},
1394 {0x00000074, 0x000000ff},
1395 {0x00000075, 0x00143400},
1396 {0x00000076, 0x08ec0800},
1397 {0x00000077, 0x040000cc},
1398 {0x00000079, 0x00000000},
1399 {0x0000007a, 0x21000409},
1400 {0x0000007c, 0x00000000},
1401 {0x0000007d, 0xe8000000},
1402 {0x0000007e, 0x044408a8},
1403 {0x0000007f, 0x00000003},
1404 {0x00000080, 0x00000000},
1405 {0x00000081, 0x01000000},
1406 {0x00000082, 0x02000000},
1407 {0x00000083, 0x00000000},
1408 {0x00000084, 0xe3f3e4f4},
1409 {0x00000085, 0x00052024},
1410 {0x00000087, 0x00000000},
1411 {0x00000088, 0x66036603},
1412 {0x00000089, 0x01000000},
1413 {0x0000008b, 0x1c0a0000},
1414 {0x0000008c, 0xff010000},
1415 {0x0000008e, 0xffffefff},
1416 {0x0000008f, 0xfff3efff},
1417 {0x00000090, 0xfff3efbf},
1418 {0x00000094, 0x00101101},
1419 {0x00000095, 0x00000fff},
1420 {0x00000096, 0x00116fff},
1421 {0x00000097, 0x60010000},
1422 {0x00000098, 0x10010000},
1423 {0x00000099, 0x00006000},
1424 {0x0000009a, 0x00001000},
1425 {0x0000009f, 0x00a17730}
1428 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1429 {0x0000006f, 0x03044000},
1430 {0x00000070, 0x0480c018},
1431 {0x00000071, 0x00000040},
1432 {0x00000072, 0x01000000},
1433 {0x00000074, 0x000000ff},
1434 {0x00000075, 0x00143400},
1435 {0x00000076, 0x08ec0800},
1436 {0x00000077, 0x040000cc},
1437 {0x00000079, 0x00000000},
1438 {0x0000007a, 0x21000409},
1439 {0x0000007c, 0x00000000},
1440 {0x0000007d, 0xe8000000},
1441 {0x0000007e, 0x044408a8},
1442 {0x0000007f, 0x00000003},
1443 {0x00000080, 0x00000000},
1444 {0x00000081, 0x01000000},
1445 {0x00000082, 0x02000000},
1446 {0x00000083, 0x00000000},
1447 {0x00000084, 0xe3f3e4f4},
1448 {0x00000085, 0x00052024},
1449 {0x00000087, 0x00000000},
1450 {0x00000088, 0x66036603},
1451 {0x00000089, 0x01000000},
1452 {0x0000008b, 0x1c0a0000},
1453 {0x0000008c, 0xff010000},
1454 {0x0000008e, 0xffffefff},
1455 {0x0000008f, 0xfff3efff},
1456 {0x00000090, 0xfff3efbf},
1457 {0x00000094, 0x00101101},
1458 {0x00000095, 0x00000fff},
1459 {0x00000096, 0x00116fff},
1460 {0x00000097, 0x60010000},
1461 {0x00000098, 0x10010000},
1462 {0x00000099, 0x00006000},
1463 {0x0000009a, 0x00001000},
1464 {0x0000009f, 0x00a07730}
1468 int si_mc_load_microcode(struct radeon_device *rdev)
1470 const __be32 *fw_data = NULL;
1471 const __le32 *new_fw_data = NULL;
1473 u32 *io_mc_regs = NULL;
1474 const __le32 *new_io_mc_regs = NULL;
1475 int i, regs_size, ucode_size;
1481 const struct mc_firmware_header_v1_0 *hdr =
1482 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1484 radeon_ucode_print_mc_hdr(&hdr->header);
1485 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1486 new_io_mc_regs = (const __le32 *)
1487 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1488 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1489 new_fw_data = (const __le32 *)
1490 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1492 ucode_size = rdev->mc_fw->size / 4;
1494 switch (rdev->family) {
1496 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1497 regs_size = TAHITI_IO_MC_REGS_SIZE;
1500 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1501 regs_size = TAHITI_IO_MC_REGS_SIZE;
1505 io_mc_regs = (u32 *)&verde_io_mc_regs;
1506 regs_size = TAHITI_IO_MC_REGS_SIZE;
1509 io_mc_regs = (u32 *)&oland_io_mc_regs;
1510 regs_size = TAHITI_IO_MC_REGS_SIZE;
1513 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1514 regs_size = TAHITI_IO_MC_REGS_SIZE;
1517 fw_data = (const __be32 *)rdev->mc_fw->data;
1520 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1523 /* reset the engine and set to writable */
1524 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1525 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1527 /* load mc io regs */
1528 for (i = 0; i < regs_size; i++) {
1530 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1531 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1533 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1534 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1537 /* load the MC ucode */
1538 for (i = 0; i < ucode_size; i++) {
1540 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1542 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1545 /* put the engine back into the active state */
1546 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1547 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1548 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1550 /* wait for training to complete */
1551 for (i = 0; i < rdev->usec_timeout; i++) {
1552 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1556 for (i = 0; i < rdev->usec_timeout; i++) {
1557 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1566 static int si_init_microcode(struct radeon_device *rdev)
1568 const char *chip_name;
1569 const char *new_chip_name;
1570 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1571 size_t smc_req_size, mc2_req_size;
1575 bool new_smc = false;
1576 bool si58_fw = false;
1580 switch (rdev->family) {
1582 chip_name = "TAHITI";
1583 new_chip_name = "tahiti";
1584 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1585 me_req_size = SI_PM4_UCODE_SIZE * 4;
1586 ce_req_size = SI_CE_UCODE_SIZE * 4;
1587 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1588 mc_req_size = SI_MC_UCODE_SIZE * 4;
1589 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1590 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1593 chip_name = "PITCAIRN";
1594 if ((rdev->pdev->revision == 0x81) &&
1595 ((rdev->pdev->device == 0x6810) ||
1596 (rdev->pdev->device == 0x6811)))
1598 new_chip_name = "pitcairn";
1599 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1600 me_req_size = SI_PM4_UCODE_SIZE * 4;
1601 ce_req_size = SI_CE_UCODE_SIZE * 4;
1602 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1603 mc_req_size = SI_MC_UCODE_SIZE * 4;
1604 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1605 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1608 chip_name = "VERDE";
1609 if (((rdev->pdev->device == 0x6820) &&
1610 ((rdev->pdev->revision == 0x81) ||
1611 (rdev->pdev->revision == 0x83))) ||
1612 ((rdev->pdev->device == 0x6821) &&
1613 ((rdev->pdev->revision == 0x83) ||
1614 (rdev->pdev->revision == 0x87))) ||
1615 ((rdev->pdev->revision == 0x87) &&
1616 ((rdev->pdev->device == 0x6823) ||
1617 (rdev->pdev->device == 0x682b))))
1619 new_chip_name = "verde";
1620 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1621 me_req_size = SI_PM4_UCODE_SIZE * 4;
1622 ce_req_size = SI_CE_UCODE_SIZE * 4;
1623 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1624 mc_req_size = SI_MC_UCODE_SIZE * 4;
1625 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1626 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1629 chip_name = "OLAND";
1630 if (((rdev->pdev->revision == 0x81) &&
1631 ((rdev->pdev->device == 0x6600) ||
1632 (rdev->pdev->device == 0x6604) ||
1633 (rdev->pdev->device == 0x6605) ||
1634 (rdev->pdev->device == 0x6610))) ||
1635 ((rdev->pdev->revision == 0x83) &&
1636 (rdev->pdev->device == 0x6610)))
1638 new_chip_name = "oland";
1639 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1640 me_req_size = SI_PM4_UCODE_SIZE * 4;
1641 ce_req_size = SI_CE_UCODE_SIZE * 4;
1642 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1643 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1644 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1647 chip_name = "HAINAN";
1648 if (((rdev->pdev->revision == 0x81) &&
1649 (rdev->pdev->device == 0x6660)) ||
1650 ((rdev->pdev->revision == 0x83) &&
1651 ((rdev->pdev->device == 0x6660) ||
1652 (rdev->pdev->device == 0x6663) ||
1653 (rdev->pdev->device == 0x6665) ||
1654 (rdev->pdev->device == 0x6667))) ||
1655 ((rdev->pdev->revision == 0xc3) &&
1656 (rdev->pdev->device == 0x6665)))
1658 new_chip_name = "hainan";
1659 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1660 me_req_size = SI_PM4_UCODE_SIZE * 4;
1661 ce_req_size = SI_CE_UCODE_SIZE * 4;
1662 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1663 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1664 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1669 /* this memory configuration requires special firmware */
1670 if (((RREG32(MC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
1673 DRM_INFO("Loading %s Microcode\n", new_chip_name);
1675 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1676 err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1678 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1679 err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1682 if (rdev->pfp_fw->size != pfp_req_size) {
1684 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1685 rdev->pfp_fw->size, fw_name);
1690 err = radeon_ucode_validate(rdev->pfp_fw);
1693 "si_cp: validation failed for firmware \"%s\"\n",
1701 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1702 err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
1704 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1705 err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
1708 if (rdev->me_fw->size != me_req_size) {
1710 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1711 rdev->me_fw->size, fw_name);
1715 err = radeon_ucode_validate(rdev->me_fw);
1718 "si_cp: validation failed for firmware \"%s\"\n",
1726 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1727 err = reject_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1729 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1730 err = reject_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1733 if (rdev->ce_fw->size != ce_req_size) {
1735 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1736 rdev->ce_fw->size, fw_name);
1740 err = radeon_ucode_validate(rdev->ce_fw);
1743 "si_cp: validation failed for firmware \"%s\"\n",
1751 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1752 err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1754 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1755 err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1758 if (rdev->rlc_fw->size != rlc_req_size) {
1760 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1761 rdev->rlc_fw->size, fw_name);
1765 err = radeon_ucode_validate(rdev->rlc_fw);
1768 "si_cp: validation failed for firmware \"%s\"\n",
1777 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/");
1779 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1780 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1782 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1783 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1785 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1786 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1790 if ((rdev->mc_fw->size != mc_req_size) &&
1791 (rdev->mc_fw->size != mc2_req_size)) {
1793 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1794 rdev->mc_fw->size, fw_name);
1797 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1799 err = radeon_ucode_validate(rdev->mc_fw);
1802 "si_cp: validation failed for firmware \"%s\"\n",
1811 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1813 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1814 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1816 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1817 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1820 "smc: error loading firmware \"%s\"\n",
1822 release_firmware(rdev->smc_fw);
1823 rdev->smc_fw = NULL;
1825 } else if (rdev->smc_fw->size != smc_req_size) {
1827 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1828 rdev->smc_fw->size, fw_name);
1832 err = radeon_ucode_validate(rdev->smc_fw);
1835 "si_cp: validation failed for firmware \"%s\"\n",
1844 rdev->new_fw = false;
1845 } else if (new_fw < 6) {
1846 printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1849 rdev->new_fw = true;
1855 "si_cp: Failed to load firmware \"%s\"\n",
1857 release_firmware(rdev->pfp_fw);
1858 rdev->pfp_fw = NULL;
1859 release_firmware(rdev->me_fw);
1861 release_firmware(rdev->ce_fw);
1863 release_firmware(rdev->rlc_fw);
1864 rdev->rlc_fw = NULL;
1865 release_firmware(rdev->mc_fw);
1867 release_firmware(rdev->smc_fw);
1868 rdev->smc_fw = NULL;
1873 /* watermark setup */
1874 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1875 struct radeon_crtc *radeon_crtc,
1876 struct drm_display_mode *mode,
1877 struct drm_display_mode *other_mode)
1879 u32 tmp, buffer_alloc, i;
1880 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1883 * There are 3 line buffers, each one shared by 2 display controllers.
1884 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1885 * the display controllers. The paritioning is done via one of four
1886 * preset allocations specified in bits 21:20:
1888 * 2 - whole lb, other crtc must be disabled
1890 /* this can get tricky if we have two large displays on a paired group
1891 * of crtcs. Ideally for multiple large displays we'd assign them to
1892 * non-linked crtcs for maximum line buffer allocation.
1894 if (radeon_crtc->base.enabled && mode) {
1899 tmp = 2; /* whole */
1907 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1908 DC_LB_MEMORY_CONFIG(tmp));
1910 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1911 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1912 for (i = 0; i < rdev->usec_timeout; i++) {
1913 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1914 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1919 if (radeon_crtc->base.enabled && mode) {
1929 /* controller not enabled, so no lb used */
1933 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1935 u32 tmp = RREG32(MC_SHARED_CHMAP);
1937 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1960 struct dce6_wm_params {
1961 u32 dram_channels; /* number of dram channels */
1962 u32 yclk; /* bandwidth per dram data pin in kHz */
1963 u32 sclk; /* engine clock in kHz */
1964 u32 disp_clk; /* display clock in kHz */
1965 u32 src_width; /* viewport width */
1966 u32 active_time; /* active display time in ns */
1967 u32 blank_time; /* blank time in ns */
1968 bool interlaced; /* mode is interlaced */
1969 fixed20_12 vsc; /* vertical scale ratio */
1970 u32 num_heads; /* number of active crtcs */
1971 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1972 u32 lb_size; /* line buffer allocated to pipe */
1973 u32 vtaps; /* vertical scaler taps */
1976 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1978 /* Calculate raw DRAM Bandwidth */
1979 fixed20_12 dram_efficiency; /* 0.7 */
1980 fixed20_12 yclk, dram_channels, bandwidth;
1983 a.full = dfixed_const(1000);
1984 yclk.full = dfixed_const(wm->yclk);
1985 yclk.full = dfixed_div(yclk, a);
1986 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1987 a.full = dfixed_const(10);
1988 dram_efficiency.full = dfixed_const(7);
1989 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1990 bandwidth.full = dfixed_mul(dram_channels, yclk);
1991 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1993 return dfixed_trunc(bandwidth);
1996 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1998 /* Calculate DRAM Bandwidth and the part allocated to display. */
1999 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2000 fixed20_12 yclk, dram_channels, bandwidth;
2003 a.full = dfixed_const(1000);
2004 yclk.full = dfixed_const(wm->yclk);
2005 yclk.full = dfixed_div(yclk, a);
2006 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2007 a.full = dfixed_const(10);
2008 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2009 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2010 bandwidth.full = dfixed_mul(dram_channels, yclk);
2011 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2013 return dfixed_trunc(bandwidth);
2016 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2018 /* Calculate the display Data return Bandwidth */
2019 fixed20_12 return_efficiency; /* 0.8 */
2020 fixed20_12 sclk, bandwidth;
2023 a.full = dfixed_const(1000);
2024 sclk.full = dfixed_const(wm->sclk);
2025 sclk.full = dfixed_div(sclk, a);
2026 a.full = dfixed_const(10);
2027 return_efficiency.full = dfixed_const(8);
2028 return_efficiency.full = dfixed_div(return_efficiency, a);
2029 a.full = dfixed_const(32);
2030 bandwidth.full = dfixed_mul(a, sclk);
2031 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2033 return dfixed_trunc(bandwidth);
2036 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2041 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2043 /* Calculate the DMIF Request Bandwidth */
2044 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2045 fixed20_12 disp_clk, sclk, bandwidth;
2046 fixed20_12 a, b1, b2;
2049 a.full = dfixed_const(1000);
2050 disp_clk.full = dfixed_const(wm->disp_clk);
2051 disp_clk.full = dfixed_div(disp_clk, a);
2052 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2053 b1.full = dfixed_mul(a, disp_clk);
2055 a.full = dfixed_const(1000);
2056 sclk.full = dfixed_const(wm->sclk);
2057 sclk.full = dfixed_div(sclk, a);
2058 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2059 b2.full = dfixed_mul(a, sclk);
2061 a.full = dfixed_const(10);
2062 disp_clk_request_efficiency.full = dfixed_const(8);
2063 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2065 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2067 a.full = dfixed_const(min_bandwidth);
2068 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2070 return dfixed_trunc(bandwidth);
2073 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2075 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2076 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2077 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2078 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2080 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2083 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2085 /* Calculate the display mode Average Bandwidth
2086 * DisplayMode should contain the source and destination dimensions,
2090 fixed20_12 line_time;
2091 fixed20_12 src_width;
2092 fixed20_12 bandwidth;
2095 a.full = dfixed_const(1000);
2096 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2097 line_time.full = dfixed_div(line_time, a);
2098 bpp.full = dfixed_const(wm->bytes_per_pixel);
2099 src_width.full = dfixed_const(wm->src_width);
2100 bandwidth.full = dfixed_mul(src_width, bpp);
2101 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2102 bandwidth.full = dfixed_div(bandwidth, line_time);
2104 return dfixed_trunc(bandwidth);
2107 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2109 /* First calcualte the latency in ns */
2110 u32 mc_latency = 2000; /* 2000 ns. */
2111 u32 available_bandwidth = dce6_available_bandwidth(wm);
2112 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2113 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2114 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2115 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2116 (wm->num_heads * cursor_line_pair_return_time);
2117 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2118 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2119 u32 tmp, dmif_size = 12288;
2122 if (wm->num_heads == 0)
2125 a.full = dfixed_const(2);
2126 b.full = dfixed_const(1);
2127 if ((wm->vsc.full > a.full) ||
2128 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2130 ((wm->vsc.full >= a.full) && wm->interlaced))
2131 max_src_lines_per_dst_line = 4;
2133 max_src_lines_per_dst_line = 2;
2135 a.full = dfixed_const(available_bandwidth);
2136 b.full = dfixed_const(wm->num_heads);
2137 a.full = dfixed_div(a, b);
2139 b.full = dfixed_const(mc_latency + 512);
2140 c.full = dfixed_const(wm->disp_clk);
2141 b.full = dfixed_div(b, c);
2143 c.full = dfixed_const(dmif_size);
2144 b.full = dfixed_div(c, b);
2146 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2148 b.full = dfixed_const(1000);
2149 c.full = dfixed_const(wm->disp_clk);
2150 b.full = dfixed_div(c, b);
2151 c.full = dfixed_const(wm->bytes_per_pixel);
2152 b.full = dfixed_mul(b, c);
2154 lb_fill_bw = min(tmp, dfixed_trunc(b));
2156 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2157 b.full = dfixed_const(1000);
2158 c.full = dfixed_const(lb_fill_bw);
2159 b.full = dfixed_div(c, b);
2160 a.full = dfixed_div(a, b);
2161 line_fill_time = dfixed_trunc(a);
2163 if (line_fill_time < wm->active_time)
2166 return latency + (line_fill_time - wm->active_time);
2170 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2172 if (dce6_average_bandwidth(wm) <=
2173 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2179 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2181 if (dce6_average_bandwidth(wm) <=
2182 (dce6_available_bandwidth(wm) / wm->num_heads))
2188 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2190 u32 lb_partitions = wm->lb_size / wm->src_width;
2191 u32 line_time = wm->active_time + wm->blank_time;
2192 u32 latency_tolerant_lines;
2196 a.full = dfixed_const(1);
2197 if (wm->vsc.full > a.full)
2198 latency_tolerant_lines = 1;
2200 if (lb_partitions <= (wm->vtaps + 1))
2201 latency_tolerant_lines = 1;
2203 latency_tolerant_lines = 2;
2206 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2208 if (dce6_latency_watermark(wm) <= latency_hiding)
2214 static void dce6_program_watermarks(struct radeon_device *rdev,
2215 struct radeon_crtc *radeon_crtc,
2216 u32 lb_size, u32 num_heads)
2218 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2219 struct dce6_wm_params wm_low, wm_high;
2223 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2224 u32 priority_a_mark = 0, priority_b_mark = 0;
2225 u32 priority_a_cnt = PRIORITY_OFF;
2226 u32 priority_b_cnt = PRIORITY_OFF;
2227 u32 tmp, arb_control3;
2230 if (radeon_crtc->base.enabled && num_heads && mode) {
2231 pixel_period = 1000000 / (u32)mode->clock;
2232 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2236 if (rdev->family == CHIP_ARUBA)
2237 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2239 dram_channels = si_get_number_of_dram_channels(rdev);
2241 /* watermark for high clocks */
2242 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2244 radeon_dpm_get_mclk(rdev, false) * 10;
2246 radeon_dpm_get_sclk(rdev, false) * 10;
2248 wm_high.yclk = rdev->pm.current_mclk * 10;
2249 wm_high.sclk = rdev->pm.current_sclk * 10;
2252 wm_high.disp_clk = mode->clock;
2253 wm_high.src_width = mode->crtc_hdisplay;
2254 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2255 wm_high.blank_time = line_time - wm_high.active_time;
2256 wm_high.interlaced = false;
2257 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2258 wm_high.interlaced = true;
2259 wm_high.vsc = radeon_crtc->vsc;
2261 if (radeon_crtc->rmx_type != RMX_OFF)
2263 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2264 wm_high.lb_size = lb_size;
2265 wm_high.dram_channels = dram_channels;
2266 wm_high.num_heads = num_heads;
2268 /* watermark for low clocks */
2269 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2271 radeon_dpm_get_mclk(rdev, true) * 10;
2273 radeon_dpm_get_sclk(rdev, true) * 10;
2275 wm_low.yclk = rdev->pm.current_mclk * 10;
2276 wm_low.sclk = rdev->pm.current_sclk * 10;
2279 wm_low.disp_clk = mode->clock;
2280 wm_low.src_width = mode->crtc_hdisplay;
2281 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2282 wm_low.blank_time = line_time - wm_low.active_time;
2283 wm_low.interlaced = false;
2284 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2285 wm_low.interlaced = true;
2286 wm_low.vsc = radeon_crtc->vsc;
2288 if (radeon_crtc->rmx_type != RMX_OFF)
2290 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2291 wm_low.lb_size = lb_size;
2292 wm_low.dram_channels = dram_channels;
2293 wm_low.num_heads = num_heads;
2295 /* set for high clocks */
2296 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2297 /* set for low clocks */
2298 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2300 /* possibly force display priority to high */
2301 /* should really do this at mode validation time... */
2302 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2303 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2304 !dce6_check_latency_hiding(&wm_high) ||
2305 (rdev->disp_priority == 2)) {
2306 DRM_DEBUG_KMS("force priority to high\n");
2307 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2308 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2310 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2311 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2312 !dce6_check_latency_hiding(&wm_low) ||
2313 (rdev->disp_priority == 2)) {
2314 DRM_DEBUG_KMS("force priority to high\n");
2315 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2316 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2319 a.full = dfixed_const(1000);
2320 b.full = dfixed_const(mode->clock);
2321 b.full = dfixed_div(b, a);
2322 c.full = dfixed_const(latency_watermark_a);
2323 c.full = dfixed_mul(c, b);
2324 c.full = dfixed_mul(c, radeon_crtc->hsc);
2325 c.full = dfixed_div(c, a);
2326 a.full = dfixed_const(16);
2327 c.full = dfixed_div(c, a);
2328 priority_a_mark = dfixed_trunc(c);
2329 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2331 a.full = dfixed_const(1000);
2332 b.full = dfixed_const(mode->clock);
2333 b.full = dfixed_div(b, a);
2334 c.full = dfixed_const(latency_watermark_b);
2335 c.full = dfixed_mul(c, b);
2336 c.full = dfixed_mul(c, radeon_crtc->hsc);
2337 c.full = dfixed_div(c, a);
2338 a.full = dfixed_const(16);
2339 c.full = dfixed_div(c, a);
2340 priority_b_mark = dfixed_trunc(c);
2341 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2343 /* Save number of lines the linebuffer leads before the scanout */
2344 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2348 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2350 tmp &= ~LATENCY_WATERMARK_MASK(3);
2351 tmp |= LATENCY_WATERMARK_MASK(1);
2352 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2353 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2354 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2355 LATENCY_HIGH_WATERMARK(line_time)));
2357 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2358 tmp &= ~LATENCY_WATERMARK_MASK(3);
2359 tmp |= LATENCY_WATERMARK_MASK(2);
2360 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2361 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2362 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2363 LATENCY_HIGH_WATERMARK(line_time)));
2364 /* restore original selection */
2365 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2367 /* write the priority marks */
2368 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2369 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2371 /* save values for DPM */
2372 radeon_crtc->line_time = line_time;
2373 radeon_crtc->wm_high = latency_watermark_a;
2374 radeon_crtc->wm_low = latency_watermark_b;
2377 void dce6_bandwidth_update(struct radeon_device *rdev)
2379 struct drm_display_mode *mode0 = NULL;
2380 struct drm_display_mode *mode1 = NULL;
2381 u32 num_heads = 0, lb_size;
2384 if (!rdev->mode_info.mode_config_initialized)
2387 radeon_update_display_priority(rdev);
2389 for (i = 0; i < rdev->num_crtc; i++) {
2390 if (rdev->mode_info.crtcs[i]->base.enabled)
2393 for (i = 0; i < rdev->num_crtc; i += 2) {
2394 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2395 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2396 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2397 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2398 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2399 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2406 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2408 u32 *tile = rdev->config.si.tile_mode_array;
2409 const u32 num_tile_mode_states =
2410 ARRAY_SIZE(rdev->config.si.tile_mode_array);
2411 u32 reg_offset, split_equal_to_row_size;
2413 switch (rdev->config.si.mem_row_size_in_kb) {
2415 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2419 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2422 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2426 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2427 tile[reg_offset] = 0;
2429 switch(rdev->family) {
2432 /* non-AA compressed depth or any compressed stencil */
2433 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2434 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2435 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2436 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2437 NUM_BANKS(ADDR_SURF_16_BANK) |
2438 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2439 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2440 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2441 /* 2xAA/4xAA compressed depth only */
2442 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2443 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2444 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2445 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2446 NUM_BANKS(ADDR_SURF_16_BANK) |
2447 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2448 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2449 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2450 /* 8xAA compressed depth only */
2451 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2452 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2453 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2454 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2455 NUM_BANKS(ADDR_SURF_16_BANK) |
2456 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2459 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2460 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2461 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2462 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2463 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2464 NUM_BANKS(ADDR_SURF_16_BANK) |
2465 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2468 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2469 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2470 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2471 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2472 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2473 NUM_BANKS(ADDR_SURF_16_BANK) |
2474 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2475 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2476 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2477 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2478 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2479 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2480 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2481 TILE_SPLIT(split_equal_to_row_size) |
2482 NUM_BANKS(ADDR_SURF_16_BANK) |
2483 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2484 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2485 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2486 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2487 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2488 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2489 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2490 TILE_SPLIT(split_equal_to_row_size) |
2491 NUM_BANKS(ADDR_SURF_16_BANK) |
2492 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2493 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2494 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2495 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2496 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2497 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2498 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2499 TILE_SPLIT(split_equal_to_row_size) |
2500 NUM_BANKS(ADDR_SURF_16_BANK) |
2501 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2503 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2504 /* 1D and 1D Array Surfaces */
2505 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2506 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2507 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2508 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2509 NUM_BANKS(ADDR_SURF_16_BANK) |
2510 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2511 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2512 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2513 /* Displayable maps. */
2514 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2515 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2517 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2518 NUM_BANKS(ADDR_SURF_16_BANK) |
2519 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2520 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2521 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2523 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2525 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2526 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2527 NUM_BANKS(ADDR_SURF_16_BANK) |
2528 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2530 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2531 /* Display 16bpp. */
2532 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2533 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2536 NUM_BANKS(ADDR_SURF_16_BANK) |
2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2540 /* Display 32bpp. */
2541 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2542 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2544 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2545 NUM_BANKS(ADDR_SURF_16_BANK) |
2546 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2548 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2550 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2551 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2552 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2553 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2554 NUM_BANKS(ADDR_SURF_16_BANK) |
2555 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2559 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2562 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2563 NUM_BANKS(ADDR_SURF_16_BANK) |
2564 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2565 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2566 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2568 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2569 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2571 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2572 NUM_BANKS(ADDR_SURF_16_BANK) |
2573 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2574 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2575 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2577 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2578 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2579 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2580 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2581 NUM_BANKS(ADDR_SURF_16_BANK) |
2582 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2583 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2584 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2586 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2587 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2589 TILE_SPLIT(split_equal_to_row_size) |
2590 NUM_BANKS(ADDR_SURF_16_BANK) |
2591 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2595 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2599 NUM_BANKS(ADDR_SURF_16_BANK) |
2600 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2604 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2607 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2608 NUM_BANKS(ADDR_SURF_16_BANK) |
2609 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2610 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2611 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2613 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2616 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2617 NUM_BANKS(ADDR_SURF_16_BANK) |
2618 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2622 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2626 NUM_BANKS(ADDR_SURF_16_BANK) |
2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2631 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2632 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2633 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2634 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2635 NUM_BANKS(ADDR_SURF_8_BANK) |
2636 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2640 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2641 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2647 /* non-AA compressed depth or any compressed stencil */
2648 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2649 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2650 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2651 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2652 NUM_BANKS(ADDR_SURF_16_BANK) |
2653 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2654 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2655 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2656 /* 2xAA/4xAA compressed depth only */
2657 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2658 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2659 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2660 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2661 NUM_BANKS(ADDR_SURF_16_BANK) |
2662 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2663 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2664 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2665 /* 8xAA compressed depth only */
2666 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2668 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2670 NUM_BANKS(ADDR_SURF_16_BANK) |
2671 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2672 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2673 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2674 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2675 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2676 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2677 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2678 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2679 NUM_BANKS(ADDR_SURF_16_BANK) |
2680 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2683 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2684 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2685 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2686 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2687 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2688 NUM_BANKS(ADDR_SURF_16_BANK) |
2689 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2690 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2691 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2692 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2693 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2694 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2695 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2696 TILE_SPLIT(split_equal_to_row_size) |
2697 NUM_BANKS(ADDR_SURF_16_BANK) |
2698 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2701 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2702 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2703 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2704 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2705 TILE_SPLIT(split_equal_to_row_size) |
2706 NUM_BANKS(ADDR_SURF_16_BANK) |
2707 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2710 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2711 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714 TILE_SPLIT(split_equal_to_row_size) |
2715 NUM_BANKS(ADDR_SURF_16_BANK) |
2716 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2719 /* 1D and 1D Array Surfaces */
2720 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2721 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2722 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2723 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2724 NUM_BANKS(ADDR_SURF_16_BANK) |
2725 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2728 /* Displayable maps. */
2729 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2730 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2731 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2732 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2733 NUM_BANKS(ADDR_SURF_16_BANK) |
2734 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2738 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2739 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2740 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2741 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2742 NUM_BANKS(ADDR_SURF_16_BANK) |
2743 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2746 /* Display 16bpp. */
2747 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2748 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2749 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2750 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2751 NUM_BANKS(ADDR_SURF_16_BANK) |
2752 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2753 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2754 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2755 /* Display 32bpp. */
2756 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2757 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2758 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2760 NUM_BANKS(ADDR_SURF_16_BANK) |
2761 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2765 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2766 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2767 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2769 NUM_BANKS(ADDR_SURF_16_BANK) |
2770 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2771 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2772 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2774 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2775 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2776 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2777 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2778 NUM_BANKS(ADDR_SURF_16_BANK) |
2779 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2783 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2784 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2785 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2787 NUM_BANKS(ADDR_SURF_16_BANK) |
2788 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2789 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2790 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2792 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2793 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2794 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2796 NUM_BANKS(ADDR_SURF_16_BANK) |
2797 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2798 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2799 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2801 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2802 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2803 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2804 TILE_SPLIT(split_equal_to_row_size) |
2805 NUM_BANKS(ADDR_SURF_16_BANK) |
2806 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2810 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2811 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2812 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2813 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2814 NUM_BANKS(ADDR_SURF_16_BANK) |
2815 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2816 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2817 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2819 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2822 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2823 NUM_BANKS(ADDR_SURF_16_BANK) |
2824 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2828 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2831 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2832 NUM_BANKS(ADDR_SURF_16_BANK) |
2833 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2837 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2840 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2841 NUM_BANKS(ADDR_SURF_16_BANK) |
2842 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2843 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2844 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2846 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2847 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2848 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2849 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2850 NUM_BANKS(ADDR_SURF_8_BANK) |
2851 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2852 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2853 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2855 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2856 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2860 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2864 static void si_select_se_sh(struct radeon_device *rdev,
2865 u32 se_num, u32 sh_num)
2867 u32 data = INSTANCE_BROADCAST_WRITES;
2869 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2870 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2871 else if (se_num == 0xffffffff)
2872 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2873 else if (sh_num == 0xffffffff)
2874 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2876 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2877 WREG32(GRBM_GFX_INDEX, data);
2880 static u32 si_create_bitmask(u32 bit_width)
2884 for (i = 0; i < bit_width; i++) {
2891 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2895 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2897 data &= INACTIVE_CUS_MASK;
2900 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2902 data >>= INACTIVE_CUS_SHIFT;
2904 mask = si_create_bitmask(cu_per_sh);
2906 return ~data & mask;
2909 static void si_setup_spi(struct radeon_device *rdev,
2910 u32 se_num, u32 sh_per_se,
2914 u32 data, mask, active_cu;
2916 for (i = 0; i < se_num; i++) {
2917 for (j = 0; j < sh_per_se; j++) {
2918 si_select_se_sh(rdev, i, j);
2919 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2920 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2923 for (k = 0; k < 16; k++) {
2925 if (active_cu & mask) {
2927 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2933 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2936 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2937 u32 max_rb_num_per_se,
2942 data = RREG32(CC_RB_BACKEND_DISABLE);
2944 data &= BACKEND_DISABLE_MASK;
2947 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2949 data >>= BACKEND_DISABLE_SHIFT;
2951 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2956 static void si_setup_rb(struct radeon_device *rdev,
2957 u32 se_num, u32 sh_per_se,
2958 u32 max_rb_num_per_se)
2962 u32 disabled_rbs = 0;
2963 u32 enabled_rbs = 0;
2965 for (i = 0; i < se_num; i++) {
2966 for (j = 0; j < sh_per_se; j++) {
2967 si_select_se_sh(rdev, i, j);
2968 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2969 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2972 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2975 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2976 if (!(disabled_rbs & mask))
2977 enabled_rbs |= mask;
2981 rdev->config.si.backend_enable_mask = enabled_rbs;
2983 for (i = 0; i < se_num; i++) {
2984 si_select_se_sh(rdev, i, 0xffffffff);
2986 for (j = 0; j < sh_per_se; j++) {
2987 switch (enabled_rbs & 3) {
2989 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2992 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2996 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3001 WREG32(PA_SC_RASTER_CONFIG, data);
3003 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3006 static void si_gpu_init(struct radeon_device *rdev)
3008 u32 gb_addr_config = 0;
3009 u32 mc_shared_chmap, mc_arb_ramcfg;
3011 u32 hdp_host_path_cntl;
3015 switch (rdev->family) {
3017 rdev->config.si.max_shader_engines = 2;
3018 rdev->config.si.max_tile_pipes = 12;
3019 rdev->config.si.max_cu_per_sh = 8;
3020 rdev->config.si.max_sh_per_se = 2;
3021 rdev->config.si.max_backends_per_se = 4;
3022 rdev->config.si.max_texture_channel_caches = 12;
3023 rdev->config.si.max_gprs = 256;
3024 rdev->config.si.max_gs_threads = 32;
3025 rdev->config.si.max_hw_contexts = 8;
3027 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3028 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3029 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3030 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3031 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3034 rdev->config.si.max_shader_engines = 2;
3035 rdev->config.si.max_tile_pipes = 8;
3036 rdev->config.si.max_cu_per_sh = 5;
3037 rdev->config.si.max_sh_per_se = 2;
3038 rdev->config.si.max_backends_per_se = 4;
3039 rdev->config.si.max_texture_channel_caches = 8;
3040 rdev->config.si.max_gprs = 256;
3041 rdev->config.si.max_gs_threads = 32;
3042 rdev->config.si.max_hw_contexts = 8;
3044 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3045 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3046 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3047 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3048 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3052 rdev->config.si.max_shader_engines = 1;
3053 rdev->config.si.max_tile_pipes = 4;
3054 rdev->config.si.max_cu_per_sh = 5;
3055 rdev->config.si.max_sh_per_se = 2;
3056 rdev->config.si.max_backends_per_se = 4;
3057 rdev->config.si.max_texture_channel_caches = 4;
3058 rdev->config.si.max_gprs = 256;
3059 rdev->config.si.max_gs_threads = 32;
3060 rdev->config.si.max_hw_contexts = 8;
3062 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3063 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3064 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3065 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3066 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3069 rdev->config.si.max_shader_engines = 1;
3070 rdev->config.si.max_tile_pipes = 4;
3071 rdev->config.si.max_cu_per_sh = 6;
3072 rdev->config.si.max_sh_per_se = 1;
3073 rdev->config.si.max_backends_per_se = 2;
3074 rdev->config.si.max_texture_channel_caches = 4;
3075 rdev->config.si.max_gprs = 256;
3076 rdev->config.si.max_gs_threads = 16;
3077 rdev->config.si.max_hw_contexts = 8;
3079 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3080 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3081 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3082 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3083 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3086 rdev->config.si.max_shader_engines = 1;
3087 rdev->config.si.max_tile_pipes = 4;
3088 rdev->config.si.max_cu_per_sh = 5;
3089 rdev->config.si.max_sh_per_se = 1;
3090 rdev->config.si.max_backends_per_se = 1;
3091 rdev->config.si.max_texture_channel_caches = 2;
3092 rdev->config.si.max_gprs = 256;
3093 rdev->config.si.max_gs_threads = 16;
3094 rdev->config.si.max_hw_contexts = 8;
3096 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3097 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3098 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3099 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3100 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3104 /* Initialize HDP */
3105 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3106 WREG32((0x2c14 + j), 0x00000000);
3107 WREG32((0x2c18 + j), 0x00000000);
3108 WREG32((0x2c1c + j), 0x00000000);
3109 WREG32((0x2c20 + j), 0x00000000);
3110 WREG32((0x2c24 + j), 0x00000000);
3113 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3114 WREG32(SRBM_INT_CNTL, 1);
3115 WREG32(SRBM_INT_ACK, 1);
3117 evergreen_fix_pci_max_read_req_size(rdev);
3119 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3121 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3122 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3124 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3125 rdev->config.si.mem_max_burst_length_bytes = 256;
3126 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3127 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3128 if (rdev->config.si.mem_row_size_in_kb > 4)
3129 rdev->config.si.mem_row_size_in_kb = 4;
3130 /* XXX use MC settings? */
3131 rdev->config.si.shader_engine_tile_size = 32;
3132 rdev->config.si.num_gpus = 1;
3133 rdev->config.si.multi_gpu_tile_size = 64;
3135 /* fix up row size */
3136 gb_addr_config &= ~ROW_SIZE_MASK;
3137 switch (rdev->config.si.mem_row_size_in_kb) {
3140 gb_addr_config |= ROW_SIZE(0);
3143 gb_addr_config |= ROW_SIZE(1);
3146 gb_addr_config |= ROW_SIZE(2);
3150 /* setup tiling info dword. gb_addr_config is not adequate since it does
3151 * not have bank info, so create a custom tiling dword.
3152 * bits 3:0 num_pipes
3153 * bits 7:4 num_banks
3154 * bits 11:8 group_size
3155 * bits 15:12 row_size
3157 rdev->config.si.tile_config = 0;
3158 switch (rdev->config.si.num_tile_pipes) {
3160 rdev->config.si.tile_config |= (0 << 0);
3163 rdev->config.si.tile_config |= (1 << 0);
3166 rdev->config.si.tile_config |= (2 << 0);
3170 /* XXX what about 12? */
3171 rdev->config.si.tile_config |= (3 << 0);
3174 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3175 case 0: /* four banks */
3176 rdev->config.si.tile_config |= 0 << 4;
3178 case 1: /* eight banks */
3179 rdev->config.si.tile_config |= 1 << 4;
3181 case 2: /* sixteen banks */
3183 rdev->config.si.tile_config |= 2 << 4;
3186 rdev->config.si.tile_config |=
3187 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3188 rdev->config.si.tile_config |=
3189 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3191 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3192 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3193 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3194 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3195 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3196 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3197 if (rdev->has_uvd) {
3198 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3199 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3200 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3203 si_tiling_mode_table_init(rdev);
3205 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3206 rdev->config.si.max_sh_per_se,
3207 rdev->config.si.max_backends_per_se);
3209 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3210 rdev->config.si.max_sh_per_se,
3211 rdev->config.si.max_cu_per_sh);
3213 rdev->config.si.active_cus = 0;
3214 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3215 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3216 rdev->config.si.active_cus +=
3217 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3221 /* set HW defaults for 3D engine */
3222 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3223 ROQ_IB2_START(0x2b)));
3224 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3226 sx_debug_1 = RREG32(SX_DEBUG_1);
3227 WREG32(SX_DEBUG_1, sx_debug_1);
3229 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3231 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3232 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3233 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3234 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3236 WREG32(VGT_NUM_INSTANCES, 1);
3238 WREG32(CP_PERFMON_CNTL, 0);
3240 WREG32(SQ_CONFIG, 0);
3242 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3243 FORCE_EOV_MAX_REZ_CNT(255)));
3245 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3246 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3248 WREG32(VGT_GS_VERTEX_REUSE, 16);
3249 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3251 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3252 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3253 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3254 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3255 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3256 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3257 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3258 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3260 tmp = RREG32(HDP_MISC_CNTL);
3261 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3262 WREG32(HDP_MISC_CNTL, tmp);
3264 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3265 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3267 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3273 * GPU scratch registers helpers function.
3275 static void si_scratch_init(struct radeon_device *rdev)
3279 rdev->scratch.num_reg = 7;
3280 rdev->scratch.reg_base = SCRATCH_REG0;
3281 for (i = 0; i < rdev->scratch.num_reg; i++) {
3282 rdev->scratch.free[i] = true;
3283 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3287 void si_fence_ring_emit(struct radeon_device *rdev,
3288 struct radeon_fence *fence)
3290 struct radeon_ring *ring = &rdev->ring[fence->ring];
3291 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3293 /* flush read cache over gart */
3294 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3295 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3296 radeon_ring_write(ring, 0);
3297 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3298 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3299 PACKET3_TC_ACTION_ENA |
3300 PACKET3_SH_KCACHE_ACTION_ENA |
3301 PACKET3_SH_ICACHE_ACTION_ENA);
3302 radeon_ring_write(ring, 0xFFFFFFFF);
3303 radeon_ring_write(ring, 0);
3304 radeon_ring_write(ring, 10); /* poll interval */
3305 /* EVENT_WRITE_EOP - flush caches, send int */
3306 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3307 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3308 radeon_ring_write(ring, lower_32_bits(addr));
3309 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3310 radeon_ring_write(ring, fence->seq);
3311 radeon_ring_write(ring, 0);
3317 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3319 struct radeon_ring *ring = &rdev->ring[ib->ring];
3320 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3323 if (ib->is_const_ib) {
3324 /* set switch buffer packet before const IB */
3325 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3326 radeon_ring_write(ring, 0);
3328 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3331 if (ring->rptr_save_reg) {
3332 next_rptr = ring->wptr + 3 + 4 + 8;
3333 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3334 radeon_ring_write(ring, ((ring->rptr_save_reg -
3335 PACKET3_SET_CONFIG_REG_START) >> 2));
3336 radeon_ring_write(ring, next_rptr);
3337 } else if (rdev->wb.enabled) {
3338 next_rptr = ring->wptr + 5 + 4 + 8;
3339 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3340 radeon_ring_write(ring, (1 << 8));
3341 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3342 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3343 radeon_ring_write(ring, next_rptr);
3346 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3349 radeon_ring_write(ring, header);
3350 radeon_ring_write(ring,
3354 (ib->gpu_addr & 0xFFFFFFFC));
3355 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3356 radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3358 if (!ib->is_const_ib) {
3359 /* flush read cache over gart for this vmid */
3360 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3361 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3362 radeon_ring_write(ring, vm_id);
3363 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3364 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3365 PACKET3_TC_ACTION_ENA |
3366 PACKET3_SH_KCACHE_ACTION_ENA |
3367 PACKET3_SH_ICACHE_ACTION_ENA);
3368 radeon_ring_write(ring, 0xFFFFFFFF);
3369 radeon_ring_write(ring, 0);
3370 radeon_ring_write(ring, 10); /* poll interval */
3377 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3380 WREG32(CP_ME_CNTL, 0);
3382 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3383 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3384 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3385 WREG32(SCRATCH_UMSK, 0);
3386 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3387 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3388 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3393 static int si_cp_load_microcode(struct radeon_device *rdev)
3397 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3400 si_cp_enable(rdev, false);
3403 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3404 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3405 const struct gfx_firmware_header_v1_0 *ce_hdr =
3406 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3407 const struct gfx_firmware_header_v1_0 *me_hdr =
3408 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3409 const __le32 *fw_data;
3412 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3413 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3414 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3417 fw_data = (const __le32 *)
3418 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3419 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3420 WREG32(CP_PFP_UCODE_ADDR, 0);
3421 for (i = 0; i < fw_size; i++)
3422 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3423 WREG32(CP_PFP_UCODE_ADDR, 0);
3426 fw_data = (const __le32 *)
3427 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3428 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3429 WREG32(CP_CE_UCODE_ADDR, 0);
3430 for (i = 0; i < fw_size; i++)
3431 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3432 WREG32(CP_CE_UCODE_ADDR, 0);
3435 fw_data = (const __be32 *)
3436 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3437 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3438 WREG32(CP_ME_RAM_WADDR, 0);
3439 for (i = 0; i < fw_size; i++)
3440 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3441 WREG32(CP_ME_RAM_WADDR, 0);
3443 const __be32 *fw_data;
3446 fw_data = (const __be32 *)rdev->pfp_fw->data;
3447 WREG32(CP_PFP_UCODE_ADDR, 0);
3448 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3449 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3450 WREG32(CP_PFP_UCODE_ADDR, 0);
3453 fw_data = (const __be32 *)rdev->ce_fw->data;
3454 WREG32(CP_CE_UCODE_ADDR, 0);
3455 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3456 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3457 WREG32(CP_CE_UCODE_ADDR, 0);
3460 fw_data = (const __be32 *)rdev->me_fw->data;
3461 WREG32(CP_ME_RAM_WADDR, 0);
3462 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3463 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3464 WREG32(CP_ME_RAM_WADDR, 0);
3467 WREG32(CP_PFP_UCODE_ADDR, 0);
3468 WREG32(CP_CE_UCODE_ADDR, 0);
3469 WREG32(CP_ME_RAM_WADDR, 0);
3470 WREG32(CP_ME_RAM_RADDR, 0);
3474 static int si_cp_start(struct radeon_device *rdev)
3476 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3479 r = radeon_ring_lock(rdev, ring, 7 + 4);
3481 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3485 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3486 radeon_ring_write(ring, 0x1);
3487 radeon_ring_write(ring, 0x0);
3488 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3489 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3490 radeon_ring_write(ring, 0);
3491 radeon_ring_write(ring, 0);
3493 /* init the CE partitions */
3494 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3495 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3496 radeon_ring_write(ring, 0xc000);
3497 radeon_ring_write(ring, 0xe000);
3498 radeon_ring_unlock_commit(rdev, ring, false);
3500 si_cp_enable(rdev, true);
3502 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3504 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3508 /* setup clear context state */
3509 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3510 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3512 for (i = 0; i < si_default_size; i++)
3513 radeon_ring_write(ring, si_default_state[i]);
3515 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3516 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3518 /* set clear context state */
3519 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3520 radeon_ring_write(ring, 0);
3522 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3523 radeon_ring_write(ring, 0x00000316);
3524 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3525 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3527 radeon_ring_unlock_commit(rdev, ring, false);
3529 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3530 ring = &rdev->ring[i];
3531 r = radeon_ring_lock(rdev, ring, 2);
3533 /* clear the compute context state */
3534 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3535 radeon_ring_write(ring, 0);
3537 radeon_ring_unlock_commit(rdev, ring, false);
3543 static void si_cp_fini(struct radeon_device *rdev)
3545 struct radeon_ring *ring;
3546 si_cp_enable(rdev, false);
3548 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3549 radeon_ring_fini(rdev, ring);
3550 radeon_scratch_free(rdev, ring->rptr_save_reg);
3552 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3553 radeon_ring_fini(rdev, ring);
3554 radeon_scratch_free(rdev, ring->rptr_save_reg);
3556 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3557 radeon_ring_fini(rdev, ring);
3558 radeon_scratch_free(rdev, ring->rptr_save_reg);
3561 static int si_cp_resume(struct radeon_device *rdev)
3563 struct radeon_ring *ring;
3568 si_enable_gui_idle_interrupt(rdev, false);
3570 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3571 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3573 /* Set the write pointer delay */
3574 WREG32(CP_RB_WPTR_DELAY, 0);
3576 WREG32(CP_DEBUG, 0);
3577 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3579 /* ring 0 - compute and gfx */
3580 /* Set ring buffer size */
3581 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3582 rb_bufsz = order_base_2(ring->ring_size / 8);
3583 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3585 tmp |= BUF_SWAP_32BIT;
3587 WREG32(CP_RB0_CNTL, tmp);
3589 /* Initialize the ring buffer's read and write pointers */
3590 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3592 WREG32(CP_RB0_WPTR, ring->wptr);
3594 /* set the wb address whether it's enabled or not */
3595 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3596 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3598 if (rdev->wb.enabled)
3599 WREG32(SCRATCH_UMSK, 0xff);
3601 tmp |= RB_NO_UPDATE;
3602 WREG32(SCRATCH_UMSK, 0);
3606 WREG32(CP_RB0_CNTL, tmp);
3608 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3610 /* ring1 - compute only */
3611 /* Set ring buffer size */
3612 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3613 rb_bufsz = order_base_2(ring->ring_size / 8);
3614 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3616 tmp |= BUF_SWAP_32BIT;
3618 WREG32(CP_RB1_CNTL, tmp);
3620 /* Initialize the ring buffer's read and write pointers */
3621 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3623 WREG32(CP_RB1_WPTR, ring->wptr);
3625 /* set the wb address whether it's enabled or not */
3626 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3627 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3630 WREG32(CP_RB1_CNTL, tmp);
3632 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3634 /* ring2 - compute only */
3635 /* Set ring buffer size */
3636 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3637 rb_bufsz = order_base_2(ring->ring_size / 8);
3638 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3640 tmp |= BUF_SWAP_32BIT;
3642 WREG32(CP_RB2_CNTL, tmp);
3644 /* Initialize the ring buffer's read and write pointers */
3645 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3647 WREG32(CP_RB2_WPTR, ring->wptr);
3649 /* set the wb address whether it's enabled or not */
3650 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3651 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3654 WREG32(CP_RB2_CNTL, tmp);
3656 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3658 /* start the rings */
3660 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3661 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3662 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3663 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3665 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3666 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3667 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3670 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3672 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3674 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3676 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3679 si_enable_gui_idle_interrupt(rdev, true);
3681 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3682 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3687 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3693 tmp = RREG32(GRBM_STATUS);
3694 if (tmp & (PA_BUSY | SC_BUSY |
3695 BCI_BUSY | SX_BUSY |
3696 TA_BUSY | VGT_BUSY |
3698 GDS_BUSY | SPI_BUSY |
3699 IA_BUSY | IA_BUSY_NO_DMA))
3700 reset_mask |= RADEON_RESET_GFX;
3702 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3703 CP_BUSY | CP_COHERENCY_BUSY))
3704 reset_mask |= RADEON_RESET_CP;
3706 if (tmp & GRBM_EE_BUSY)
3707 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3710 tmp = RREG32(GRBM_STATUS2);
3711 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3712 reset_mask |= RADEON_RESET_RLC;
3714 /* DMA_STATUS_REG 0 */
3715 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3716 if (!(tmp & DMA_IDLE))
3717 reset_mask |= RADEON_RESET_DMA;
3719 /* DMA_STATUS_REG 1 */
3720 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3721 if (!(tmp & DMA_IDLE))
3722 reset_mask |= RADEON_RESET_DMA1;
3725 tmp = RREG32(SRBM_STATUS2);
3727 reset_mask |= RADEON_RESET_DMA;
3729 if (tmp & DMA1_BUSY)
3730 reset_mask |= RADEON_RESET_DMA1;
3733 tmp = RREG32(SRBM_STATUS);
3736 reset_mask |= RADEON_RESET_IH;
3739 reset_mask |= RADEON_RESET_SEM;
3741 if (tmp & GRBM_RQ_PENDING)
3742 reset_mask |= RADEON_RESET_GRBM;
3745 reset_mask |= RADEON_RESET_VMC;
3747 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3748 MCC_BUSY | MCD_BUSY))
3749 reset_mask |= RADEON_RESET_MC;
3751 if (evergreen_is_display_hung(rdev))
3752 reset_mask |= RADEON_RESET_DISPLAY;
3755 tmp = RREG32(VM_L2_STATUS);
3757 reset_mask |= RADEON_RESET_VMC;
3759 /* Skip MC reset as it's mostly likely not hung, just busy */
3760 if (reset_mask & RADEON_RESET_MC) {
3761 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3762 reset_mask &= ~RADEON_RESET_MC;
3768 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3770 struct evergreen_mc_save save;
3771 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3774 if (reset_mask == 0)
3777 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3779 evergreen_print_gpu_status_regs(rdev);
3780 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3781 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3782 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3783 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3792 /* Disable CP parsing/prefetching */
3793 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3795 if (reset_mask & RADEON_RESET_DMA) {
3797 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3798 tmp &= ~DMA_RB_ENABLE;
3799 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3801 if (reset_mask & RADEON_RESET_DMA1) {
3803 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3804 tmp &= ~DMA_RB_ENABLE;
3805 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3810 evergreen_mc_stop(rdev, &save);
3811 if (evergreen_mc_wait_for_idle(rdev)) {
3812 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3815 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3816 grbm_soft_reset = SOFT_RESET_CB |
3830 if (reset_mask & RADEON_RESET_CP) {
3831 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3833 srbm_soft_reset |= SOFT_RESET_GRBM;
3836 if (reset_mask & RADEON_RESET_DMA)
3837 srbm_soft_reset |= SOFT_RESET_DMA;
3839 if (reset_mask & RADEON_RESET_DMA1)
3840 srbm_soft_reset |= SOFT_RESET_DMA1;
3842 if (reset_mask & RADEON_RESET_DISPLAY)
3843 srbm_soft_reset |= SOFT_RESET_DC;
3845 if (reset_mask & RADEON_RESET_RLC)
3846 grbm_soft_reset |= SOFT_RESET_RLC;
3848 if (reset_mask & RADEON_RESET_SEM)
3849 srbm_soft_reset |= SOFT_RESET_SEM;
3851 if (reset_mask & RADEON_RESET_IH)
3852 srbm_soft_reset |= SOFT_RESET_IH;
3854 if (reset_mask & RADEON_RESET_GRBM)
3855 srbm_soft_reset |= SOFT_RESET_GRBM;
3857 if (reset_mask & RADEON_RESET_VMC)
3858 srbm_soft_reset |= SOFT_RESET_VMC;
3860 if (reset_mask & RADEON_RESET_MC)
3861 srbm_soft_reset |= SOFT_RESET_MC;
3863 if (grbm_soft_reset) {
3864 tmp = RREG32(GRBM_SOFT_RESET);
3865 tmp |= grbm_soft_reset;
3866 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3867 WREG32(GRBM_SOFT_RESET, tmp);
3868 tmp = RREG32(GRBM_SOFT_RESET);
3872 tmp &= ~grbm_soft_reset;
3873 WREG32(GRBM_SOFT_RESET, tmp);
3874 tmp = RREG32(GRBM_SOFT_RESET);
3877 if (srbm_soft_reset) {
3878 tmp = RREG32(SRBM_SOFT_RESET);
3879 tmp |= srbm_soft_reset;
3880 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3881 WREG32(SRBM_SOFT_RESET, tmp);
3882 tmp = RREG32(SRBM_SOFT_RESET);
3886 tmp &= ~srbm_soft_reset;
3887 WREG32(SRBM_SOFT_RESET, tmp);
3888 tmp = RREG32(SRBM_SOFT_RESET);
3891 /* Wait a little for things to settle down */
3894 evergreen_mc_resume(rdev, &save);
3897 evergreen_print_gpu_status_regs(rdev);
3900 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3904 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3905 tmp |= SPLL_BYPASS_EN;
3906 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3908 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3909 tmp |= SPLL_CTLREQ_CHG;
3910 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3912 for (i = 0; i < rdev->usec_timeout; i++) {
3913 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3918 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3919 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3920 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3922 tmp = RREG32(MPLL_CNTL_MODE);
3923 tmp &= ~MPLL_MCLK_SEL;
3924 WREG32(MPLL_CNTL_MODE, tmp);
3927 static void si_spll_powerdown(struct radeon_device *rdev)
3931 tmp = RREG32(SPLL_CNTL_MODE);
3932 tmp |= SPLL_SW_DIR_CONTROL;
3933 WREG32(SPLL_CNTL_MODE, tmp);
3935 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3937 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3939 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3941 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3943 tmp = RREG32(SPLL_CNTL_MODE);
3944 tmp &= ~SPLL_SW_DIR_CONTROL;
3945 WREG32(SPLL_CNTL_MODE, tmp);
3948 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3950 struct evergreen_mc_save save;
3953 dev_info(rdev->dev, "GPU pci config reset\n");
3961 /* Disable CP parsing/prefetching */
3962 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3964 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3965 tmp &= ~DMA_RB_ENABLE;
3966 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3968 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3969 tmp &= ~DMA_RB_ENABLE;
3970 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3971 /* XXX other engines? */
3973 /* halt the rlc, disable cp internal ints */
3978 /* disable mem access */
3979 evergreen_mc_stop(rdev, &save);
3980 if (evergreen_mc_wait_for_idle(rdev)) {
3981 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3984 /* set mclk/sclk to bypass */
3985 si_set_clk_bypass_mode(rdev);
3986 /* powerdown spll */
3987 si_spll_powerdown(rdev);
3989 pci_clear_master(rdev->pdev);
3991 radeon_pci_config_reset(rdev);
3992 /* wait for asic to come out of reset */
3993 for (i = 0; i < rdev->usec_timeout; i++) {
3994 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4000 int si_asic_reset(struct radeon_device *rdev, bool hard)
4005 si_gpu_pci_config_reset(rdev);
4009 reset_mask = si_gpu_check_soft_reset(rdev);
4012 r600_set_bios_scratch_engine_hung(rdev, true);
4014 /* try soft reset */
4015 si_gpu_soft_reset(rdev, reset_mask);
4017 reset_mask = si_gpu_check_soft_reset(rdev);
4019 /* try pci config reset */
4020 if (reset_mask && radeon_hard_reset)
4021 si_gpu_pci_config_reset(rdev);
4023 reset_mask = si_gpu_check_soft_reset(rdev);
4026 r600_set_bios_scratch_engine_hung(rdev, false);
4032 * si_gfx_is_lockup - Check if the GFX engine is locked up
4034 * @rdev: radeon_device pointer
4035 * @ring: radeon_ring structure holding ring information
4037 * Check if the GFX engine is locked up.
4038 * Returns true if the engine appears to be locked up, false if not.
4040 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4042 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4044 if (!(reset_mask & (RADEON_RESET_GFX |
4045 RADEON_RESET_COMPUTE |
4046 RADEON_RESET_CP))) {
4047 radeon_ring_lockup_update(rdev, ring);
4050 return radeon_ring_test_lockup(rdev, ring);
4054 static void si_mc_program(struct radeon_device *rdev)
4056 struct evergreen_mc_save save;
4060 /* Initialize HDP */
4061 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4062 WREG32((0x2c14 + j), 0x00000000);
4063 WREG32((0x2c18 + j), 0x00000000);
4064 WREG32((0x2c1c + j), 0x00000000);
4065 WREG32((0x2c20 + j), 0x00000000);
4066 WREG32((0x2c24 + j), 0x00000000);
4068 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4070 evergreen_mc_stop(rdev, &save);
4071 if (radeon_mc_wait_for_idle(rdev)) {
4072 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4074 if (!ASIC_IS_NODCE(rdev))
4075 /* Lockout access through VGA aperture*/
4076 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4077 /* Update configuration */
4078 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4079 rdev->mc.vram_start >> 12);
4080 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4081 rdev->mc.vram_end >> 12);
4082 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4083 rdev->vram_scratch.gpu_addr >> 12);
4084 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4085 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4086 WREG32(MC_VM_FB_LOCATION, tmp);
4087 /* XXX double check these! */
4088 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4089 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4090 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4091 WREG32(MC_VM_AGP_BASE, 0);
4092 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4093 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4094 if (radeon_mc_wait_for_idle(rdev)) {
4095 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4097 evergreen_mc_resume(rdev, &save);
4098 if (!ASIC_IS_NODCE(rdev)) {
4099 /* we need to own VRAM, so turn off the VGA renderer here
4100 * to stop it overwriting our objects */
4101 rv515_vga_render_disable(rdev);
4105 void si_vram_gtt_location(struct radeon_device *rdev,
4106 struct radeon_mc *mc)
4108 if (mc->mc_vram_size > 0xFFC0000000ULL) {
4109 /* leave room for at least 1024M GTT */
4110 dev_warn(rdev->dev, "limiting VRAM\n");
4111 mc->real_vram_size = 0xFFC0000000ULL;
4112 mc->mc_vram_size = 0xFFC0000000ULL;
4114 radeon_vram_location(rdev, &rdev->mc, 0);
4115 rdev->mc.gtt_base_align = 0;
4116 radeon_gtt_location(rdev, mc);
4119 static int si_mc_init(struct radeon_device *rdev)
4122 int chansize, numchan;
4124 /* Get VRAM informations */
4125 rdev->mc.vram_is_ddr = true;
4126 tmp = RREG32(MC_ARB_RAMCFG);
4127 if (tmp & CHANSIZE_OVERRIDE) {
4129 } else if (tmp & CHANSIZE_MASK) {
4134 tmp = RREG32(MC_SHARED_CHMAP);
4135 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4165 rdev->mc.vram_width = numchan * chansize;
4166 /* Could aper size report 0 ? */
4167 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4168 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4169 /* size in MB on si */
4170 tmp = RREG32(CONFIG_MEMSIZE);
4171 /* some boards may have garbage in the upper 16 bits */
4172 if (tmp & 0xffff0000) {
4173 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4177 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4178 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4179 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4180 si_vram_gtt_location(rdev, &rdev->mc);
4181 radeon_update_bandwidth_info(rdev);
4189 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4191 /* flush hdp cache */
4192 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4194 /* bits 0-15 are the VM contexts0-15 */
4195 WREG32(VM_INVALIDATE_REQUEST, 1);
4198 static int si_pcie_gart_enable(struct radeon_device *rdev)
4202 if (rdev->gart.robj == NULL) {
4203 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4206 r = radeon_gart_table_vram_pin(rdev);
4209 /* Setup TLB control */
4210 WREG32(MC_VM_MX_L1_TLB_CNTL,
4213 ENABLE_L1_FRAGMENT_PROCESSING |
4214 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4215 ENABLE_ADVANCED_DRIVER_MODEL |
4216 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4217 /* Setup L2 cache */
4218 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4219 ENABLE_L2_FRAGMENT_PROCESSING |
4220 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4221 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4222 EFFECTIVE_L2_QUEUE_SIZE(7) |
4223 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4224 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4225 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4227 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4228 /* setup context0 */
4229 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4230 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4231 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4232 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4233 (u32)(rdev->dummy_page.addr >> 12));
4234 WREG32(VM_CONTEXT0_CNTL2, 0);
4235 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4236 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4242 /* empty context1-15 */
4243 /* set vm size, must be a multiple of 4 */
4244 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4245 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4246 /* Assign the pt base to something valid for now; the pts used for
4247 * the VMs are determined by the application and setup and assigned
4248 * on the fly in the vm part of radeon_gart.c
4250 for (i = 1; i < 16; i++) {
4252 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4253 rdev->vm_manager.saved_table_addr[i]);
4255 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4256 rdev->vm_manager.saved_table_addr[i]);
4259 /* enable context1-15 */
4260 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4261 (u32)(rdev->dummy_page.addr >> 12));
4262 WREG32(VM_CONTEXT1_CNTL2, 4);
4263 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4264 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4265 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4266 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4267 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4268 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4269 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4270 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4271 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4272 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4273 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4274 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4275 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4276 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4278 si_pcie_gart_tlb_flush(rdev);
4279 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4280 (unsigned)(rdev->mc.gtt_size >> 20),
4281 (unsigned long long)rdev->gart.table_addr);
4282 rdev->gart.ready = true;
4286 static void si_pcie_gart_disable(struct radeon_device *rdev)
4290 for (i = 1; i < 16; ++i) {
4293 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4295 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4296 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4299 /* Disable all tables */
4300 WREG32(VM_CONTEXT0_CNTL, 0);
4301 WREG32(VM_CONTEXT1_CNTL, 0);
4302 /* Setup TLB control */
4303 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4304 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4305 /* Setup L2 cache */
4306 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4307 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4308 EFFECTIVE_L2_QUEUE_SIZE(7) |
4309 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4310 WREG32(VM_L2_CNTL2, 0);
4311 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4312 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4313 radeon_gart_table_vram_unpin(rdev);
4316 static void si_pcie_gart_fini(struct radeon_device *rdev)
4318 si_pcie_gart_disable(rdev);
4319 radeon_gart_table_vram_free(rdev);
4320 radeon_gart_fini(rdev);
4324 static bool si_vm_reg_valid(u32 reg)
4326 /* context regs are fine */
4330 /* shader regs are also fine */
4331 if (reg >= 0xB000 && reg < 0xC000)
4334 /* check config regs */
4336 case GRBM_GFX_INDEX:
4337 case CP_STRMOUT_CNTL:
4338 case VGT_VTX_VECT_EJECT_REG:
4339 case VGT_CACHE_INVALIDATION:
4340 case VGT_ESGS_RING_SIZE:
4341 case VGT_GSVS_RING_SIZE:
4342 case VGT_GS_VERTEX_REUSE:
4343 case VGT_PRIMITIVE_TYPE:
4344 case VGT_INDEX_TYPE:
4345 case VGT_NUM_INDICES:
4346 case VGT_NUM_INSTANCES:
4347 case VGT_TF_RING_SIZE:
4348 case VGT_HS_OFFCHIP_PARAM:
4349 case VGT_TF_MEMORY_BASE:
4351 case PA_SU_LINE_STIPPLE_VALUE:
4352 case PA_SC_LINE_STIPPLE_STATE:
4355 case SPI_STATIC_THREAD_MGMT_1:
4356 case SPI_STATIC_THREAD_MGMT_2:
4357 case SPI_STATIC_THREAD_MGMT_3:
4358 case SPI_PS_MAX_WAVE_ID:
4359 case SPI_CONFIG_CNTL:
4360 case SPI_CONFIG_CNTL_1:
4362 case TA_CS_BC_BASE_ADDR:
4365 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4370 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4371 u32 *ib, struct radeon_cs_packet *pkt)
4373 switch (pkt->opcode) {
4375 case PACKET3_SET_BASE:
4376 case PACKET3_SET_CE_DE_COUNTERS:
4377 case PACKET3_LOAD_CONST_RAM:
4378 case PACKET3_WRITE_CONST_RAM:
4379 case PACKET3_WRITE_CONST_RAM_OFFSET:
4380 case PACKET3_DUMP_CONST_RAM:
4381 case PACKET3_INCREMENT_CE_COUNTER:
4382 case PACKET3_WAIT_ON_DE_COUNTER:
4383 case PACKET3_CE_WRITE:
4386 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4392 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4394 u32 start_reg, reg, i;
4395 u32 command = ib[idx + 4];
4396 u32 info = ib[idx + 1];
4397 u32 idx_value = ib[idx];
4398 if (command & PACKET3_CP_DMA_CMD_SAS) {
4399 /* src address space is register */
4400 if (((info & 0x60000000) >> 29) == 0) {
4401 start_reg = idx_value << 2;
4402 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4404 if (!si_vm_reg_valid(reg)) {
4405 DRM_ERROR("CP DMA Bad SRC register\n");
4409 for (i = 0; i < (command & 0x1fffff); i++) {
4410 reg = start_reg + (4 * i);
4411 if (!si_vm_reg_valid(reg)) {
4412 DRM_ERROR("CP DMA Bad SRC register\n");
4419 if (command & PACKET3_CP_DMA_CMD_DAS) {
4420 /* dst address space is register */
4421 if (((info & 0x00300000) >> 20) == 0) {
4422 start_reg = ib[idx + 2];
4423 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4425 if (!si_vm_reg_valid(reg)) {
4426 DRM_ERROR("CP DMA Bad DST register\n");
4430 for (i = 0; i < (command & 0x1fffff); i++) {
4431 reg = start_reg + (4 * i);
4432 if (!si_vm_reg_valid(reg)) {
4433 DRM_ERROR("CP DMA Bad DST register\n");
4443 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4444 u32 *ib, struct radeon_cs_packet *pkt)
4447 u32 idx = pkt->idx + 1;
4448 u32 idx_value = ib[idx];
4449 u32 start_reg, end_reg, reg, i;
4451 switch (pkt->opcode) {
4453 case PACKET3_SET_BASE:
4454 case PACKET3_CLEAR_STATE:
4455 case PACKET3_INDEX_BUFFER_SIZE:
4456 case PACKET3_DISPATCH_DIRECT:
4457 case PACKET3_DISPATCH_INDIRECT:
4458 case PACKET3_ALLOC_GDS:
4459 case PACKET3_WRITE_GDS_RAM:
4460 case PACKET3_ATOMIC_GDS:
4461 case PACKET3_ATOMIC:
4462 case PACKET3_OCCLUSION_QUERY:
4463 case PACKET3_SET_PREDICATION:
4464 case PACKET3_COND_EXEC:
4465 case PACKET3_PRED_EXEC:
4466 case PACKET3_DRAW_INDIRECT:
4467 case PACKET3_DRAW_INDEX_INDIRECT:
4468 case PACKET3_INDEX_BASE:
4469 case PACKET3_DRAW_INDEX_2:
4470 case PACKET3_CONTEXT_CONTROL:
4471 case PACKET3_INDEX_TYPE:
4472 case PACKET3_DRAW_INDIRECT_MULTI:
4473 case PACKET3_DRAW_INDEX_AUTO:
4474 case PACKET3_DRAW_INDEX_IMMD:
4475 case PACKET3_NUM_INSTANCES:
4476 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4477 case PACKET3_STRMOUT_BUFFER_UPDATE:
4478 case PACKET3_DRAW_INDEX_OFFSET_2:
4479 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4480 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4481 case PACKET3_MPEG_INDEX:
4482 case PACKET3_WAIT_REG_MEM:
4483 case PACKET3_MEM_WRITE:
4484 case PACKET3_PFP_SYNC_ME:
4485 case PACKET3_SURFACE_SYNC:
4486 case PACKET3_EVENT_WRITE:
4487 case PACKET3_EVENT_WRITE_EOP:
4488 case PACKET3_EVENT_WRITE_EOS:
4489 case PACKET3_SET_CONTEXT_REG:
4490 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4491 case PACKET3_SET_SH_REG:
4492 case PACKET3_SET_SH_REG_OFFSET:
4493 case PACKET3_INCREMENT_DE_COUNTER:
4494 case PACKET3_WAIT_ON_CE_COUNTER:
4495 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4496 case PACKET3_ME_WRITE:
4498 case PACKET3_COPY_DATA:
4499 if ((idx_value & 0xf00) == 0) {
4500 reg = ib[idx + 3] * 4;
4501 if (!si_vm_reg_valid(reg))
4505 case PACKET3_WRITE_DATA:
4506 if ((idx_value & 0xf00) == 0) {
4507 start_reg = ib[idx + 1] * 4;
4508 if (idx_value & 0x10000) {
4509 if (!si_vm_reg_valid(start_reg))
4512 for (i = 0; i < (pkt->count - 2); i++) {
4513 reg = start_reg + (4 * i);
4514 if (!si_vm_reg_valid(reg))
4520 case PACKET3_COND_WRITE:
4521 if (idx_value & 0x100) {
4522 reg = ib[idx + 5] * 4;
4523 if (!si_vm_reg_valid(reg))
4527 case PACKET3_COPY_DW:
4528 if (idx_value & 0x2) {
4529 reg = ib[idx + 3] * 4;
4530 if (!si_vm_reg_valid(reg))
4534 case PACKET3_SET_CONFIG_REG:
4535 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4536 end_reg = 4 * pkt->count + start_reg - 4;
4537 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4538 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4539 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4540 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4543 for (i = 0; i < pkt->count; i++) {
4544 reg = start_reg + (4 * i);
4545 if (!si_vm_reg_valid(reg))
4549 case PACKET3_CP_DMA:
4550 r = si_vm_packet3_cp_dma_check(ib, idx);
4555 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4561 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4562 u32 *ib, struct radeon_cs_packet *pkt)
4565 u32 idx = pkt->idx + 1;
4566 u32 idx_value = ib[idx];
4567 u32 start_reg, reg, i;
4569 switch (pkt->opcode) {
4571 case PACKET3_SET_BASE:
4572 case PACKET3_CLEAR_STATE:
4573 case PACKET3_DISPATCH_DIRECT:
4574 case PACKET3_DISPATCH_INDIRECT:
4575 case PACKET3_ALLOC_GDS:
4576 case PACKET3_WRITE_GDS_RAM:
4577 case PACKET3_ATOMIC_GDS:
4578 case PACKET3_ATOMIC:
4579 case PACKET3_OCCLUSION_QUERY:
4580 case PACKET3_SET_PREDICATION:
4581 case PACKET3_COND_EXEC:
4582 case PACKET3_PRED_EXEC:
4583 case PACKET3_CONTEXT_CONTROL:
4584 case PACKET3_STRMOUT_BUFFER_UPDATE:
4585 case PACKET3_WAIT_REG_MEM:
4586 case PACKET3_MEM_WRITE:
4587 case PACKET3_PFP_SYNC_ME:
4588 case PACKET3_SURFACE_SYNC:
4589 case PACKET3_EVENT_WRITE:
4590 case PACKET3_EVENT_WRITE_EOP:
4591 case PACKET3_EVENT_WRITE_EOS:
4592 case PACKET3_SET_CONTEXT_REG:
4593 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4594 case PACKET3_SET_SH_REG:
4595 case PACKET3_SET_SH_REG_OFFSET:
4596 case PACKET3_INCREMENT_DE_COUNTER:
4597 case PACKET3_WAIT_ON_CE_COUNTER:
4598 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4599 case PACKET3_ME_WRITE:
4601 case PACKET3_COPY_DATA:
4602 if ((idx_value & 0xf00) == 0) {
4603 reg = ib[idx + 3] * 4;
4604 if (!si_vm_reg_valid(reg))
4608 case PACKET3_WRITE_DATA:
4609 if ((idx_value & 0xf00) == 0) {
4610 start_reg = ib[idx + 1] * 4;
4611 if (idx_value & 0x10000) {
4612 if (!si_vm_reg_valid(start_reg))
4615 for (i = 0; i < (pkt->count - 2); i++) {
4616 reg = start_reg + (4 * i);
4617 if (!si_vm_reg_valid(reg))
4623 case PACKET3_COND_WRITE:
4624 if (idx_value & 0x100) {
4625 reg = ib[idx + 5] * 4;
4626 if (!si_vm_reg_valid(reg))
4630 case PACKET3_COPY_DW:
4631 if (idx_value & 0x2) {
4632 reg = ib[idx + 3] * 4;
4633 if (!si_vm_reg_valid(reg))
4637 case PACKET3_CP_DMA:
4638 r = si_vm_packet3_cp_dma_check(ib, idx);
4643 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4649 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4653 struct radeon_cs_packet pkt;
4657 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4658 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4661 case RADEON_PACKET_TYPE0:
4662 dev_err(rdev->dev, "Packet0 not allowed!\n");
4665 case RADEON_PACKET_TYPE2:
4668 case RADEON_PACKET_TYPE3:
4669 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4670 if (ib->is_const_ib)
4671 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4674 case RADEON_RING_TYPE_GFX_INDEX:
4675 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4677 case CAYMAN_RING_TYPE_CP1_INDEX:
4678 case CAYMAN_RING_TYPE_CP2_INDEX:
4679 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4682 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4687 idx += pkt.count + 2;
4690 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4695 for (i = 0; i < ib->length_dw; i++) {
4697 printk("\t0x%08x <---\n", ib->ptr[i]);
4699 printk("\t0x%08x\n", ib->ptr[i]);
4703 } while (idx < ib->length_dw);
4711 int si_vm_init(struct radeon_device *rdev)
4714 rdev->vm_manager.nvm = 16;
4715 /* base offset of vram pages */
4716 rdev->vm_manager.vram_base_offset = 0;
4721 void si_vm_fini(struct radeon_device *rdev)
4726 * si_vm_decode_fault - print human readable fault info
4728 * @rdev: radeon_device pointer
4729 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4730 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4732 * Print human readable fault information (SI).
4734 static void si_vm_decode_fault(struct radeon_device *rdev,
4735 u32 status, u32 addr)
4737 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4738 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4739 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4742 if (rdev->family == CHIP_TAHITI) {
4983 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4984 protections, vmid, addr,
4985 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4989 void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
4990 unsigned vm_id, uint64_t pd_addr)
4992 /* write new base address */
4993 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4994 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4995 WRITE_DATA_DST_SEL(0)));
4998 radeon_ring_write(ring,
4999 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5001 radeon_ring_write(ring,
5002 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5004 radeon_ring_write(ring, 0);
5005 radeon_ring_write(ring, pd_addr >> 12);
5007 /* flush hdp cache */
5008 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5009 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5010 WRITE_DATA_DST_SEL(0)));
5011 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5012 radeon_ring_write(ring, 0);
5013 radeon_ring_write(ring, 0x1);
5015 /* bits 0-15 are the VM contexts0-15 */
5016 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5017 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5018 WRITE_DATA_DST_SEL(0)));
5019 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5020 radeon_ring_write(ring, 0);
5021 radeon_ring_write(ring, 1 << vm_id);
5023 /* wait for the invalidate to complete */
5024 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5025 radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */
5026 WAIT_REG_MEM_ENGINE(0))); /* me */
5027 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5028 radeon_ring_write(ring, 0);
5029 radeon_ring_write(ring, 0); /* ref */
5030 radeon_ring_write(ring, 0); /* mask */
5031 radeon_ring_write(ring, 0x20); /* poll interval */
5033 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5034 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5035 radeon_ring_write(ring, 0x0);
5039 * Power and clock gating
5041 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5045 for (i = 0; i < rdev->usec_timeout; i++) {
5046 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5051 for (i = 0; i < rdev->usec_timeout; i++) {
5052 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5058 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5061 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5066 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5068 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5069 WREG32(CP_INT_CNTL_RING0, tmp);
5072 /* read a gfx register */
5073 tmp = RREG32(DB_DEPTH_INFO);
5075 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5076 for (i = 0; i < rdev->usec_timeout; i++) {
5077 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5084 static void si_set_uvd_dcm(struct radeon_device *rdev,
5089 tmp = RREG32(UVD_CGC_CTRL);
5090 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5091 tmp |= DCM | CG_DT(1) | CLK_OD(4);
5095 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5101 WREG32(UVD_CGC_CTRL, tmp);
5102 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5105 void si_init_uvd_internal_cg(struct radeon_device *rdev)
5107 bool hw_mode = true;
5110 si_set_uvd_dcm(rdev, false);
5112 u32 tmp = RREG32(UVD_CGC_CTRL);
5114 WREG32(UVD_CGC_CTRL, tmp);
5118 static u32 si_halt_rlc(struct radeon_device *rdev)
5122 orig = data = RREG32(RLC_CNTL);
5124 if (data & RLC_ENABLE) {
5125 data &= ~RLC_ENABLE;
5126 WREG32(RLC_CNTL, data);
5128 si_wait_for_rlc_serdes(rdev);
5134 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5138 tmp = RREG32(RLC_CNTL);
5140 WREG32(RLC_CNTL, rlc);
5143 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5147 orig = data = RREG32(DMA_PG);
5148 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5149 data |= PG_CNTL_ENABLE;
5151 data &= ~PG_CNTL_ENABLE;
5153 WREG32(DMA_PG, data);
5156 static void si_init_dma_pg(struct radeon_device *rdev)
5160 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5161 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5163 for (tmp = 0; tmp < 5; tmp++)
5164 WREG32(DMA_PGFSM_WRITE, 0);
5167 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5172 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5173 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5174 WREG32(RLC_TTOP_D, tmp);
5176 tmp = RREG32(RLC_PG_CNTL);
5177 tmp |= GFX_PG_ENABLE;
5178 WREG32(RLC_PG_CNTL, tmp);
5180 tmp = RREG32(RLC_AUTO_PG_CTRL);
5182 WREG32(RLC_AUTO_PG_CTRL, tmp);
5184 tmp = RREG32(RLC_AUTO_PG_CTRL);
5186 WREG32(RLC_AUTO_PG_CTRL, tmp);
5188 tmp = RREG32(DB_RENDER_CONTROL);
5192 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5196 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5198 tmp = RREG32(RLC_PG_CNTL);
5200 WREG32(RLC_PG_CNTL, tmp);
5202 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5204 tmp = RREG32(RLC_AUTO_PG_CTRL);
5206 tmp &= ~GRBM_REG_SGIT_MASK;
5207 tmp |= GRBM_REG_SGIT(0x700);
5208 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5209 WREG32(RLC_AUTO_PG_CTRL, tmp);
5212 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5214 u32 mask = 0, tmp, tmp1;
5217 si_select_se_sh(rdev, se, sh);
5218 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5219 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5220 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5227 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5232 return (~tmp) & mask;
5235 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5237 u32 i, j, k, active_cu_number = 0;
5238 u32 mask, counter, cu_bitmap;
5241 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5242 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5246 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5247 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5255 active_cu_number += counter;
5256 tmp |= (cu_bitmap << (i * 16 + j * 8));
5260 WREG32(RLC_PG_AO_CU_MASK, tmp);
5262 tmp = RREG32(RLC_MAX_PG_CU);
5263 tmp &= ~MAX_PU_CU_MASK;
5264 tmp |= MAX_PU_CU(active_cu_number);
5265 WREG32(RLC_MAX_PG_CU, tmp);
5268 static void si_enable_cgcg(struct radeon_device *rdev,
5271 u32 data, orig, tmp;
5273 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5275 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5276 si_enable_gui_idle_interrupt(rdev, true);
5278 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5280 tmp = si_halt_rlc(rdev);
5282 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5283 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5284 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5286 si_wait_for_rlc_serdes(rdev);
5288 si_update_rlc(rdev, tmp);
5290 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5292 data |= CGCG_EN | CGLS_EN;
5294 si_enable_gui_idle_interrupt(rdev, false);
5296 RREG32(CB_CGTT_SCLK_CTRL);
5297 RREG32(CB_CGTT_SCLK_CTRL);
5298 RREG32(CB_CGTT_SCLK_CTRL);
5299 RREG32(CB_CGTT_SCLK_CTRL);
5301 data &= ~(CGCG_EN | CGLS_EN);
5305 WREG32(RLC_CGCG_CGLS_CTRL, data);
5308 static void si_enable_mgcg(struct radeon_device *rdev,
5311 u32 data, orig, tmp = 0;
5313 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5314 orig = data = RREG32(CGTS_SM_CTRL_REG);
5317 WREG32(CGTS_SM_CTRL_REG, data);
5319 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5320 orig = data = RREG32(CP_MEM_SLP_CNTL);
5321 data |= CP_MEM_LS_EN;
5323 WREG32(CP_MEM_SLP_CNTL, data);
5326 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5329 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5331 tmp = si_halt_rlc(rdev);
5333 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5334 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5335 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5337 si_update_rlc(rdev, tmp);
5339 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5342 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5344 data = RREG32(CP_MEM_SLP_CNTL);
5345 if (data & CP_MEM_LS_EN) {
5346 data &= ~CP_MEM_LS_EN;
5347 WREG32(CP_MEM_SLP_CNTL, data);
5349 orig = data = RREG32(CGTS_SM_CTRL_REG);
5350 data |= LS_OVERRIDE | OVERRIDE;
5352 WREG32(CGTS_SM_CTRL_REG, data);
5354 tmp = si_halt_rlc(rdev);
5356 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5357 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5358 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5360 si_update_rlc(rdev, tmp);
5364 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5367 u32 orig, data, tmp;
5369 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5370 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5372 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5374 orig = data = RREG32(UVD_CGC_CTRL);
5377 WREG32(UVD_CGC_CTRL, data);
5379 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5380 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5382 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5384 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5386 orig = data = RREG32(UVD_CGC_CTRL);
5389 WREG32(UVD_CGC_CTRL, data);
5391 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5392 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5396 static const u32 mc_cg_registers[] =
5409 static void si_enable_mc_ls(struct radeon_device *rdev,
5415 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5416 orig = data = RREG32(mc_cg_registers[i]);
5417 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5418 data |= MC_LS_ENABLE;
5420 data &= ~MC_LS_ENABLE;
5422 WREG32(mc_cg_registers[i], data);
5426 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5432 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5433 orig = data = RREG32(mc_cg_registers[i]);
5434 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5435 data |= MC_CG_ENABLE;
5437 data &= ~MC_CG_ENABLE;
5439 WREG32(mc_cg_registers[i], data);
5443 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5446 u32 orig, data, offset;
5449 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5450 for (i = 0; i < 2; i++) {
5452 offset = DMA0_REGISTER_OFFSET;
5454 offset = DMA1_REGISTER_OFFSET;
5455 orig = data = RREG32(DMA_POWER_CNTL + offset);
5456 data &= ~MEM_POWER_OVERRIDE;
5458 WREG32(DMA_POWER_CNTL + offset, data);
5459 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5462 for (i = 0; i < 2; i++) {
5464 offset = DMA0_REGISTER_OFFSET;
5466 offset = DMA1_REGISTER_OFFSET;
5467 orig = data = RREG32(DMA_POWER_CNTL + offset);
5468 data |= MEM_POWER_OVERRIDE;
5470 WREG32(DMA_POWER_CNTL + offset, data);
5472 orig = data = RREG32(DMA_CLK_CTRL + offset);
5475 WREG32(DMA_CLK_CTRL + offset, data);
5480 static void si_enable_bif_mgls(struct radeon_device *rdev,
5485 orig = data = RREG32_PCIE(PCIE_CNTL2);
5487 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5488 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5489 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5491 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5492 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5495 WREG32_PCIE(PCIE_CNTL2, data);
5498 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5503 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5505 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5506 data &= ~CLOCK_GATING_DIS;
5508 data |= CLOCK_GATING_DIS;
5511 WREG32(HDP_HOST_PATH_CNTL, data);
5514 static void si_enable_hdp_ls(struct radeon_device *rdev,
5519 orig = data = RREG32(HDP_MEM_POWER_LS);
5521 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5522 data |= HDP_LS_ENABLE;
5524 data &= ~HDP_LS_ENABLE;
5527 WREG32(HDP_MEM_POWER_LS, data);
5530 static void si_update_cg(struct radeon_device *rdev,
5531 u32 block, bool enable)
5533 if (block & RADEON_CG_BLOCK_GFX) {
5534 si_enable_gui_idle_interrupt(rdev, false);
5535 /* order matters! */
5537 si_enable_mgcg(rdev, true);
5538 si_enable_cgcg(rdev, true);
5540 si_enable_cgcg(rdev, false);
5541 si_enable_mgcg(rdev, false);
5543 si_enable_gui_idle_interrupt(rdev, true);
5546 if (block & RADEON_CG_BLOCK_MC) {
5547 si_enable_mc_mgcg(rdev, enable);
5548 si_enable_mc_ls(rdev, enable);
5551 if (block & RADEON_CG_BLOCK_SDMA) {
5552 si_enable_dma_mgcg(rdev, enable);
5555 if (block & RADEON_CG_BLOCK_BIF) {
5556 si_enable_bif_mgls(rdev, enable);
5559 if (block & RADEON_CG_BLOCK_UVD) {
5560 if (rdev->has_uvd) {
5561 si_enable_uvd_mgcg(rdev, enable);
5565 if (block & RADEON_CG_BLOCK_HDP) {
5566 si_enable_hdp_mgcg(rdev, enable);
5567 si_enable_hdp_ls(rdev, enable);
5571 static void si_init_cg(struct radeon_device *rdev)
5573 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5574 RADEON_CG_BLOCK_MC |
5575 RADEON_CG_BLOCK_SDMA |
5576 RADEON_CG_BLOCK_BIF |
5577 RADEON_CG_BLOCK_HDP), true);
5578 if (rdev->has_uvd) {
5579 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5580 si_init_uvd_internal_cg(rdev);
5584 static void si_fini_cg(struct radeon_device *rdev)
5586 if (rdev->has_uvd) {
5587 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5589 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5590 RADEON_CG_BLOCK_MC |
5591 RADEON_CG_BLOCK_SDMA |
5592 RADEON_CG_BLOCK_BIF |
5593 RADEON_CG_BLOCK_HDP), false);
5596 u32 si_get_csb_size(struct radeon_device *rdev)
5599 const struct cs_section_def *sect = NULL;
5600 const struct cs_extent_def *ext = NULL;
5602 if (rdev->rlc.cs_data == NULL)
5605 /* begin clear state */
5607 /* context control state */
5610 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5611 for (ext = sect->section; ext->extent != NULL; ++ext) {
5612 if (sect->id == SECT_CONTEXT)
5613 count += 2 + ext->reg_count;
5618 /* pa_sc_raster_config */
5620 /* end clear state */
5628 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5631 const struct cs_section_def *sect = NULL;
5632 const struct cs_extent_def *ext = NULL;
5634 if (rdev->rlc.cs_data == NULL)
5639 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5640 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5642 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5643 buffer[count++] = cpu_to_le32(0x80000000);
5644 buffer[count++] = cpu_to_le32(0x80000000);
5646 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5647 for (ext = sect->section; ext->extent != NULL; ++ext) {
5648 if (sect->id == SECT_CONTEXT) {
5650 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5651 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5652 for (i = 0; i < ext->reg_count; i++)
5653 buffer[count++] = cpu_to_le32(ext->extent[i]);
5660 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5661 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5662 switch (rdev->family) {
5665 buffer[count++] = cpu_to_le32(0x2a00126a);
5668 buffer[count++] = cpu_to_le32(0x0000124a);
5671 buffer[count++] = cpu_to_le32(0x00000082);
5674 buffer[count++] = cpu_to_le32(0x00000000);
5677 buffer[count++] = cpu_to_le32(0x00000000);
5681 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5682 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5684 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5685 buffer[count++] = cpu_to_le32(0);
5688 static void si_init_pg(struct radeon_device *rdev)
5690 if (rdev->pg_flags) {
5691 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5692 si_init_dma_pg(rdev);
5694 si_init_ao_cu_mask(rdev);
5695 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5696 si_init_gfx_cgpg(rdev);
5698 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5699 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5701 si_enable_dma_pg(rdev, true);
5702 si_enable_gfx_cgpg(rdev, true);
5704 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5705 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5709 static void si_fini_pg(struct radeon_device *rdev)
5711 if (rdev->pg_flags) {
5712 si_enable_dma_pg(rdev, false);
5713 si_enable_gfx_cgpg(rdev, false);
5720 void si_rlc_reset(struct radeon_device *rdev)
5722 u32 tmp = RREG32(GRBM_SOFT_RESET);
5724 tmp |= SOFT_RESET_RLC;
5725 WREG32(GRBM_SOFT_RESET, tmp);
5727 tmp &= ~SOFT_RESET_RLC;
5728 WREG32(GRBM_SOFT_RESET, tmp);
5732 static void si_rlc_stop(struct radeon_device *rdev)
5734 WREG32(RLC_CNTL, 0);
5736 si_enable_gui_idle_interrupt(rdev, false);
5738 si_wait_for_rlc_serdes(rdev);
5741 static void si_rlc_start(struct radeon_device *rdev)
5743 WREG32(RLC_CNTL, RLC_ENABLE);
5745 si_enable_gui_idle_interrupt(rdev, true);
5750 static bool si_lbpw_supported(struct radeon_device *rdev)
5754 /* Enable LBPW only for DDR3 */
5755 tmp = RREG32(MC_SEQ_MISC0);
5756 if ((tmp & 0xF0000000) == 0xB0000000)
5761 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5765 tmp = RREG32(RLC_LB_CNTL);
5767 tmp |= LOAD_BALANCE_ENABLE;
5769 tmp &= ~LOAD_BALANCE_ENABLE;
5770 WREG32(RLC_LB_CNTL, tmp);
5773 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5774 WREG32(SPI_LB_CU_MASK, 0x00ff);
5778 static int si_rlc_resume(struct radeon_device *rdev)
5793 WREG32(RLC_RL_BASE, 0);
5794 WREG32(RLC_RL_SIZE, 0);
5795 WREG32(RLC_LB_CNTL, 0);
5796 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5797 WREG32(RLC_LB_CNTR_INIT, 0);
5798 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5800 WREG32(RLC_MC_CNTL, 0);
5801 WREG32(RLC_UCODE_CNTL, 0);
5804 const struct rlc_firmware_header_v1_0 *hdr =
5805 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5806 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5807 const __le32 *fw_data = (const __le32 *)
5808 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5810 radeon_ucode_print_rlc_hdr(&hdr->header);
5812 for (i = 0; i < fw_size; i++) {
5813 WREG32(RLC_UCODE_ADDR, i);
5814 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5817 const __be32 *fw_data =
5818 (const __be32 *)rdev->rlc_fw->data;
5819 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5820 WREG32(RLC_UCODE_ADDR, i);
5821 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5824 WREG32(RLC_UCODE_ADDR, 0);
5826 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5833 static void si_enable_interrupts(struct radeon_device *rdev)
5835 u32 ih_cntl = RREG32(IH_CNTL);
5836 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5838 ih_cntl |= ENABLE_INTR;
5839 ih_rb_cntl |= IH_RB_ENABLE;
5840 WREG32(IH_CNTL, ih_cntl);
5841 WREG32(IH_RB_CNTL, ih_rb_cntl);
5842 rdev->ih.enabled = true;
5845 static void si_disable_interrupts(struct radeon_device *rdev)
5847 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5848 u32 ih_cntl = RREG32(IH_CNTL);
5850 ih_rb_cntl &= ~IH_RB_ENABLE;
5851 ih_cntl &= ~ENABLE_INTR;
5852 WREG32(IH_RB_CNTL, ih_rb_cntl);
5853 WREG32(IH_CNTL, ih_cntl);
5854 /* set rptr, wptr to 0 */
5855 WREG32(IH_RB_RPTR, 0);
5856 WREG32(IH_RB_WPTR, 0);
5857 rdev->ih.enabled = false;
5861 static void si_disable_interrupt_state(struct radeon_device *rdev)
5865 tmp = RREG32(CP_INT_CNTL_RING0) &
5866 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5867 WREG32(CP_INT_CNTL_RING0, tmp);
5868 WREG32(CP_INT_CNTL_RING1, 0);
5869 WREG32(CP_INT_CNTL_RING2, 0);
5870 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5871 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5872 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5873 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5874 WREG32(GRBM_INT_CNTL, 0);
5875 WREG32(SRBM_INT_CNTL, 0);
5876 if (rdev->num_crtc >= 2) {
5877 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5878 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5880 if (rdev->num_crtc >= 4) {
5881 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5882 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5884 if (rdev->num_crtc >= 6) {
5885 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5886 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5889 if (rdev->num_crtc >= 2) {
5890 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5891 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5893 if (rdev->num_crtc >= 4) {
5894 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5895 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5897 if (rdev->num_crtc >= 6) {
5898 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5899 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5902 if (!ASIC_IS_NODCE(rdev)) {
5903 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5905 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5906 WREG32(DC_HPD1_INT_CONTROL, tmp);
5907 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5908 WREG32(DC_HPD2_INT_CONTROL, tmp);
5909 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5910 WREG32(DC_HPD3_INT_CONTROL, tmp);
5911 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5912 WREG32(DC_HPD4_INT_CONTROL, tmp);
5913 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5914 WREG32(DC_HPD5_INT_CONTROL, tmp);
5915 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5916 WREG32(DC_HPD6_INT_CONTROL, tmp);
5920 static int si_irq_init(struct radeon_device *rdev)
5924 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5927 ret = r600_ih_ring_alloc(rdev);
5932 si_disable_interrupts(rdev);
5935 ret = si_rlc_resume(rdev);
5937 r600_ih_ring_fini(rdev);
5941 /* setup interrupt control */
5942 /* set dummy read address to dummy page address */
5943 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
5944 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5945 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5946 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5948 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5949 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5950 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5951 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5953 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5954 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5956 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5957 IH_WPTR_OVERFLOW_CLEAR |
5960 if (rdev->wb.enabled)
5961 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5963 /* set the writeback address whether it's enabled or not */
5964 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5965 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5967 WREG32(IH_RB_CNTL, ih_rb_cntl);
5969 /* set rptr, wptr to 0 */
5970 WREG32(IH_RB_RPTR, 0);
5971 WREG32(IH_RB_WPTR, 0);
5973 /* Default settings for IH_CNTL (disabled at first) */
5974 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5975 /* RPTR_REARM only works if msi's are enabled */
5976 if (rdev->msi_enabled)
5977 ih_cntl |= RPTR_REARM;
5978 WREG32(IH_CNTL, ih_cntl);
5980 /* force the active interrupt state to all disabled */
5981 si_disable_interrupt_state(rdev);
5983 pci_set_master(rdev->pdev);
5986 si_enable_interrupts(rdev);
5991 int si_irq_set(struct radeon_device *rdev)
5994 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5995 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5996 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5997 u32 grbm_int_cntl = 0;
5998 u32 dma_cntl, dma_cntl1;
5999 u32 thermal_int = 0;
6001 if (!rdev->irq.installed) {
6002 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6005 /* don't enable anything if the ih is disabled */
6006 if (!rdev->ih.enabled) {
6007 si_disable_interrupts(rdev);
6008 /* force the active interrupt state to all disabled */
6009 si_disable_interrupt_state(rdev);
6013 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6014 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6016 if (!ASIC_IS_NODCE(rdev)) {
6017 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6018 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6019 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6020 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6021 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6022 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6025 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6026 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6028 thermal_int = RREG32(CG_THERMAL_INT) &
6029 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6031 /* enable CP interrupts on all rings */
6032 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6033 DRM_DEBUG("si_irq_set: sw int gfx\n");
6034 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6036 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6037 DRM_DEBUG("si_irq_set: sw int cp1\n");
6038 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6040 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6041 DRM_DEBUG("si_irq_set: sw int cp2\n");
6042 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6044 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6045 DRM_DEBUG("si_irq_set: sw int dma\n");
6046 dma_cntl |= TRAP_ENABLE;
6049 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6050 DRM_DEBUG("si_irq_set: sw int dma1\n");
6051 dma_cntl1 |= TRAP_ENABLE;
6053 if (rdev->irq.crtc_vblank_int[0] ||
6054 atomic_read(&rdev->irq.pflip[0])) {
6055 DRM_DEBUG("si_irq_set: vblank 0\n");
6056 crtc1 |= VBLANK_INT_MASK;
6058 if (rdev->irq.crtc_vblank_int[1] ||
6059 atomic_read(&rdev->irq.pflip[1])) {
6060 DRM_DEBUG("si_irq_set: vblank 1\n");
6061 crtc2 |= VBLANK_INT_MASK;
6063 if (rdev->irq.crtc_vblank_int[2] ||
6064 atomic_read(&rdev->irq.pflip[2])) {
6065 DRM_DEBUG("si_irq_set: vblank 2\n");
6066 crtc3 |= VBLANK_INT_MASK;
6068 if (rdev->irq.crtc_vblank_int[3] ||
6069 atomic_read(&rdev->irq.pflip[3])) {
6070 DRM_DEBUG("si_irq_set: vblank 3\n");
6071 crtc4 |= VBLANK_INT_MASK;
6073 if (rdev->irq.crtc_vblank_int[4] ||
6074 atomic_read(&rdev->irq.pflip[4])) {
6075 DRM_DEBUG("si_irq_set: vblank 4\n");
6076 crtc5 |= VBLANK_INT_MASK;
6078 if (rdev->irq.crtc_vblank_int[5] ||
6079 atomic_read(&rdev->irq.pflip[5])) {
6080 DRM_DEBUG("si_irq_set: vblank 5\n");
6081 crtc6 |= VBLANK_INT_MASK;
6083 if (rdev->irq.hpd[0]) {
6084 DRM_DEBUG("si_irq_set: hpd 1\n");
6085 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6087 if (rdev->irq.hpd[1]) {
6088 DRM_DEBUG("si_irq_set: hpd 2\n");
6089 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6091 if (rdev->irq.hpd[2]) {
6092 DRM_DEBUG("si_irq_set: hpd 3\n");
6093 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6095 if (rdev->irq.hpd[3]) {
6096 DRM_DEBUG("si_irq_set: hpd 4\n");
6097 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6099 if (rdev->irq.hpd[4]) {
6100 DRM_DEBUG("si_irq_set: hpd 5\n");
6101 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6103 if (rdev->irq.hpd[5]) {
6104 DRM_DEBUG("si_irq_set: hpd 6\n");
6105 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6108 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6109 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6110 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6112 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6113 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6115 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6117 if (rdev->irq.dpm_thermal) {
6118 DRM_DEBUG("dpm thermal\n");
6119 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6122 if (rdev->num_crtc >= 2) {
6123 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6124 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6126 if (rdev->num_crtc >= 4) {
6127 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6128 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6130 if (rdev->num_crtc >= 6) {
6131 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6132 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6135 if (rdev->num_crtc >= 2) {
6136 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6137 GRPH_PFLIP_INT_MASK);
6138 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6139 GRPH_PFLIP_INT_MASK);
6141 if (rdev->num_crtc >= 4) {
6142 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6143 GRPH_PFLIP_INT_MASK);
6144 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6145 GRPH_PFLIP_INT_MASK);
6147 if (rdev->num_crtc >= 6) {
6148 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6149 GRPH_PFLIP_INT_MASK);
6150 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6151 GRPH_PFLIP_INT_MASK);
6154 if (!ASIC_IS_NODCE(rdev)) {
6155 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6156 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6157 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6158 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6159 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6160 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6163 WREG32(CG_THERMAL_INT, thermal_int);
6166 RREG32(SRBM_STATUS);
6171 static inline void si_irq_ack(struct radeon_device *rdev)
6175 if (ASIC_IS_NODCE(rdev))
6178 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6179 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6180 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6181 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6182 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6183 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6184 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6185 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6186 if (rdev->num_crtc >= 4) {
6187 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6188 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6190 if (rdev->num_crtc >= 6) {
6191 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6192 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6195 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6196 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6197 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6198 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6199 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6200 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6201 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6202 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6203 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6204 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6205 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6206 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6208 if (rdev->num_crtc >= 4) {
6209 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6210 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6211 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6212 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6213 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6214 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6215 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6216 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6217 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6218 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6219 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6220 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6223 if (rdev->num_crtc >= 6) {
6224 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6225 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6226 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6227 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6228 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6229 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6230 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6231 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6232 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6233 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6234 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6235 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6238 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6239 tmp = RREG32(DC_HPD1_INT_CONTROL);
6240 tmp |= DC_HPDx_INT_ACK;
6241 WREG32(DC_HPD1_INT_CONTROL, tmp);
6243 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6244 tmp = RREG32(DC_HPD2_INT_CONTROL);
6245 tmp |= DC_HPDx_INT_ACK;
6246 WREG32(DC_HPD2_INT_CONTROL, tmp);
6248 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6249 tmp = RREG32(DC_HPD3_INT_CONTROL);
6250 tmp |= DC_HPDx_INT_ACK;
6251 WREG32(DC_HPD3_INT_CONTROL, tmp);
6253 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6254 tmp = RREG32(DC_HPD4_INT_CONTROL);
6255 tmp |= DC_HPDx_INT_ACK;
6256 WREG32(DC_HPD4_INT_CONTROL, tmp);
6258 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6259 tmp = RREG32(DC_HPD5_INT_CONTROL);
6260 tmp |= DC_HPDx_INT_ACK;
6261 WREG32(DC_HPD5_INT_CONTROL, tmp);
6263 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6264 tmp = RREG32(DC_HPD6_INT_CONTROL);
6265 tmp |= DC_HPDx_INT_ACK;
6266 WREG32(DC_HPD6_INT_CONTROL, tmp);
6269 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
6270 tmp = RREG32(DC_HPD1_INT_CONTROL);
6271 tmp |= DC_HPDx_RX_INT_ACK;
6272 WREG32(DC_HPD1_INT_CONTROL, tmp);
6274 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
6275 tmp = RREG32(DC_HPD2_INT_CONTROL);
6276 tmp |= DC_HPDx_RX_INT_ACK;
6277 WREG32(DC_HPD2_INT_CONTROL, tmp);
6279 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
6280 tmp = RREG32(DC_HPD3_INT_CONTROL);
6281 tmp |= DC_HPDx_RX_INT_ACK;
6282 WREG32(DC_HPD3_INT_CONTROL, tmp);
6284 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
6285 tmp = RREG32(DC_HPD4_INT_CONTROL);
6286 tmp |= DC_HPDx_RX_INT_ACK;
6287 WREG32(DC_HPD4_INT_CONTROL, tmp);
6289 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
6290 tmp = RREG32(DC_HPD5_INT_CONTROL);
6291 tmp |= DC_HPDx_RX_INT_ACK;
6292 WREG32(DC_HPD5_INT_CONTROL, tmp);
6294 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
6295 tmp = RREG32(DC_HPD6_INT_CONTROL);
6296 tmp |= DC_HPDx_RX_INT_ACK;
6297 WREG32(DC_HPD6_INT_CONTROL, tmp);
6301 static void si_irq_disable(struct radeon_device *rdev)
6303 si_disable_interrupts(rdev);
6304 /* Wait and acknowledge irq */
6307 si_disable_interrupt_state(rdev);
6310 static void si_irq_suspend(struct radeon_device *rdev)
6312 si_irq_disable(rdev);
6316 static void si_irq_fini(struct radeon_device *rdev)
6318 si_irq_suspend(rdev);
6319 r600_ih_ring_fini(rdev);
6322 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6326 if (rdev->wb.enabled)
6327 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6329 wptr = RREG32(IH_RB_WPTR);
6331 if (wptr & RB_OVERFLOW) {
6332 wptr &= ~RB_OVERFLOW;
6333 /* When a ring buffer overflow happen start parsing interrupt
6334 * from the last not overwritten vector (wptr + 16). Hopefully
6335 * this should allow us to catchup.
6337 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6338 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6339 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6340 tmp = RREG32(IH_RB_CNTL);
6341 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6342 WREG32(IH_RB_CNTL, tmp);
6344 return (wptr & rdev->ih.ptr_mask);
6348 * Each IV ring entry is 128 bits:
6349 * [7:0] - interrupt source id
6351 * [59:32] - interrupt source data
6352 * [63:60] - reserved
6355 * [127:80] - reserved
6357 int si_irq_process(struct radeon_device *rdev)
6361 u32 src_id, src_data, ring_id;
6363 bool queue_hotplug = false;
6364 bool queue_dp = false;
6365 bool queue_thermal = false;
6368 if (!rdev->ih.enabled || rdev->shutdown)
6371 wptr = si_get_ih_wptr(rdev);
6374 /* is somebody else already processing irqs? */
6375 if (atomic_xchg(&rdev->ih.lock, 1))
6378 rptr = rdev->ih.rptr;
6379 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6381 /* Order reading of wptr vs. reading of IH ring data */
6384 /* display interrupts */
6387 while (rptr != wptr) {
6388 /* wptr/rptr are in bytes! */
6389 ring_index = rptr / 4;
6390 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6391 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6392 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6395 case 1: /* D1 vblank/vline */
6397 case 0: /* D1 vblank */
6398 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
6399 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6401 if (rdev->irq.crtc_vblank_int[0]) {
6402 drm_handle_vblank(rdev->ddev, 0);
6403 rdev->pm.vblank_sync = true;
6404 wake_up(&rdev->irq.vblank_queue);
6406 if (atomic_read(&rdev->irq.pflip[0]))
6407 radeon_crtc_handle_vblank(rdev, 0);
6408 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6409 DRM_DEBUG("IH: D1 vblank\n");
6412 case 1: /* D1 vline */
6413 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
6414 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6416 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6417 DRM_DEBUG("IH: D1 vline\n");
6421 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6425 case 2: /* D2 vblank/vline */
6427 case 0: /* D2 vblank */
6428 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
6429 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6431 if (rdev->irq.crtc_vblank_int[1]) {
6432 drm_handle_vblank(rdev->ddev, 1);
6433 rdev->pm.vblank_sync = true;
6434 wake_up(&rdev->irq.vblank_queue);
6436 if (atomic_read(&rdev->irq.pflip[1]))
6437 radeon_crtc_handle_vblank(rdev, 1);
6438 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6439 DRM_DEBUG("IH: D2 vblank\n");
6442 case 1: /* D2 vline */
6443 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
6444 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6446 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6447 DRM_DEBUG("IH: D2 vline\n");
6451 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6455 case 3: /* D3 vblank/vline */
6457 case 0: /* D3 vblank */
6458 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
6459 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6461 if (rdev->irq.crtc_vblank_int[2]) {
6462 drm_handle_vblank(rdev->ddev, 2);
6463 rdev->pm.vblank_sync = true;
6464 wake_up(&rdev->irq.vblank_queue);
6466 if (atomic_read(&rdev->irq.pflip[2]))
6467 radeon_crtc_handle_vblank(rdev, 2);
6468 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6469 DRM_DEBUG("IH: D3 vblank\n");
6472 case 1: /* D3 vline */
6473 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
6474 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6476 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6477 DRM_DEBUG("IH: D3 vline\n");
6481 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6485 case 4: /* D4 vblank/vline */
6487 case 0: /* D4 vblank */
6488 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
6489 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6491 if (rdev->irq.crtc_vblank_int[3]) {
6492 drm_handle_vblank(rdev->ddev, 3);
6493 rdev->pm.vblank_sync = true;
6494 wake_up(&rdev->irq.vblank_queue);
6496 if (atomic_read(&rdev->irq.pflip[3]))
6497 radeon_crtc_handle_vblank(rdev, 3);
6498 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6499 DRM_DEBUG("IH: D4 vblank\n");
6502 case 1: /* D4 vline */
6503 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
6504 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6506 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6507 DRM_DEBUG("IH: D4 vline\n");
6511 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6515 case 5: /* D5 vblank/vline */
6517 case 0: /* D5 vblank */
6518 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
6519 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6521 if (rdev->irq.crtc_vblank_int[4]) {
6522 drm_handle_vblank(rdev->ddev, 4);
6523 rdev->pm.vblank_sync = true;
6524 wake_up(&rdev->irq.vblank_queue);
6526 if (atomic_read(&rdev->irq.pflip[4]))
6527 radeon_crtc_handle_vblank(rdev, 4);
6528 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6529 DRM_DEBUG("IH: D5 vblank\n");
6532 case 1: /* D5 vline */
6533 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
6534 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6536 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6537 DRM_DEBUG("IH: D5 vline\n");
6541 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6545 case 6: /* D6 vblank/vline */
6547 case 0: /* D6 vblank */
6548 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
6549 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6551 if (rdev->irq.crtc_vblank_int[5]) {
6552 drm_handle_vblank(rdev->ddev, 5);
6553 rdev->pm.vblank_sync = true;
6554 wake_up(&rdev->irq.vblank_queue);
6556 if (atomic_read(&rdev->irq.pflip[5]))
6557 radeon_crtc_handle_vblank(rdev, 5);
6558 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6559 DRM_DEBUG("IH: D6 vblank\n");
6562 case 1: /* D6 vline */
6563 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
6564 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6566 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6567 DRM_DEBUG("IH: D6 vline\n");
6571 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6575 case 8: /* D1 page flip */
6576 case 10: /* D2 page flip */
6577 case 12: /* D3 page flip */
6578 case 14: /* D4 page flip */
6579 case 16: /* D5 page flip */
6580 case 18: /* D6 page flip */
6581 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6582 if (radeon_use_pflipirq > 0)
6583 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6585 case 42: /* HPD hotplug */
6588 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
6589 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6591 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6592 queue_hotplug = true;
6593 DRM_DEBUG("IH: HPD1\n");
6597 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
6598 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6600 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6601 queue_hotplug = true;
6602 DRM_DEBUG("IH: HPD2\n");
6606 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
6607 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6609 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6610 queue_hotplug = true;
6611 DRM_DEBUG("IH: HPD3\n");
6615 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
6616 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6618 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6619 queue_hotplug = true;
6620 DRM_DEBUG("IH: HPD4\n");
6624 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
6625 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6627 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6628 queue_hotplug = true;
6629 DRM_DEBUG("IH: HPD5\n");
6633 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
6634 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6636 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6637 queue_hotplug = true;
6638 DRM_DEBUG("IH: HPD6\n");
6642 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
6643 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6645 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
6647 DRM_DEBUG("IH: HPD_RX 1\n");
6651 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
6652 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6654 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
6656 DRM_DEBUG("IH: HPD_RX 2\n");
6660 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
6661 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6663 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
6665 DRM_DEBUG("IH: HPD_RX 3\n");
6669 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
6670 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6672 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
6674 DRM_DEBUG("IH: HPD_RX 4\n");
6678 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
6679 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6681 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
6683 DRM_DEBUG("IH: HPD_RX 5\n");
6687 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
6688 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6690 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
6692 DRM_DEBUG("IH: HPD_RX 6\n");
6696 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6701 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6702 WREG32(SRBM_INT_ACK, 0x1);
6705 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6706 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6710 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6711 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6712 /* reset addr and status */
6713 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6714 if (addr == 0x0 && status == 0x0)
6716 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6717 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6719 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6721 si_vm_decode_fault(rdev, status, addr);
6723 case 176: /* RINGID0 CP_INT */
6724 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6726 case 177: /* RINGID1 CP_INT */
6727 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6729 case 178: /* RINGID2 CP_INT */
6730 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6732 case 181: /* CP EOP event */
6733 DRM_DEBUG("IH: CP EOP\n");
6736 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6739 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6742 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6746 case 224: /* DMA trap event */
6747 DRM_DEBUG("IH: DMA trap\n");
6748 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6750 case 230: /* thermal low to high */
6751 DRM_DEBUG("IH: thermal low to high\n");
6752 rdev->pm.dpm.thermal.high_to_low = false;
6753 queue_thermal = true;
6755 case 231: /* thermal high to low */
6756 DRM_DEBUG("IH: thermal high to low\n");
6757 rdev->pm.dpm.thermal.high_to_low = true;
6758 queue_thermal = true;
6760 case 233: /* GUI IDLE */
6761 DRM_DEBUG("IH: GUI idle\n");
6763 case 244: /* DMA trap event */
6764 DRM_DEBUG("IH: DMA1 trap\n");
6765 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6768 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6772 /* wptr/rptr are in bytes! */
6774 rptr &= rdev->ih.ptr_mask;
6775 WREG32(IH_RB_RPTR, rptr);
6778 schedule_work(&rdev->dp_work);
6780 schedule_delayed_work(&rdev->hotplug_work, 0);
6781 if (queue_thermal && rdev->pm.dpm_enabled)
6782 schedule_work(&rdev->pm.dpm.thermal.work);
6783 rdev->ih.rptr = rptr;
6784 atomic_set(&rdev->ih.lock, 0);
6786 /* make sure wptr hasn't changed while processing */
6787 wptr = si_get_ih_wptr(rdev);
6795 * startup/shutdown callbacks
6797 static void si_uvd_init(struct radeon_device *rdev)
6804 r = radeon_uvd_init(rdev);
6806 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6808 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6809 * to early fails uvd_v2_2_resume() and thus nothing happens
6810 * there. So it is pointless to try to go through that code
6811 * hence why we disable uvd here.
6816 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6817 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6820 static void si_uvd_start(struct radeon_device *rdev)
6827 r = uvd_v2_2_resume(rdev);
6829 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6832 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6834 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6840 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6843 static void si_uvd_resume(struct radeon_device *rdev)
6845 struct radeon_ring *ring;
6848 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6851 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6852 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
6854 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6857 r = uvd_v1_0_init(rdev);
6859 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6864 static void si_vce_init(struct radeon_device *rdev)
6871 r = radeon_vce_init(rdev);
6873 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6875 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6876 * to early fails si_vce_start() and thus nothing happens
6877 * there. So it is pointless to try to go through that code
6878 * hence why we disable vce here.
6883 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6884 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6885 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6886 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6889 static void si_vce_start(struct radeon_device *rdev)
6896 r = radeon_vce_resume(rdev);
6898 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6901 r = vce_v1_0_resume(rdev);
6903 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6906 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6908 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6911 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6913 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6919 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6920 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6923 static void si_vce_resume(struct radeon_device *rdev)
6925 struct radeon_ring *ring;
6928 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6931 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6932 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6934 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6937 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6938 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6940 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6943 r = vce_v1_0_init(rdev);
6945 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6950 static int si_startup(struct radeon_device *rdev)
6952 struct radeon_ring *ring;
6955 /* enable pcie gen2/3 link */
6956 si_pcie_gen3_enable(rdev);
6958 si_program_aspm(rdev);
6960 /* scratch needs to be initialized before MC */
6961 r = r600_vram_scratch_init(rdev);
6965 si_mc_program(rdev);
6967 if (!rdev->pm.dpm_enabled) {
6968 r = si_mc_load_microcode(rdev);
6970 DRM_ERROR("Failed to load MC firmware!\n");
6975 r = si_pcie_gart_enable(rdev);
6980 /* allocate rlc buffers */
6981 if (rdev->family == CHIP_VERDE) {
6982 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6983 rdev->rlc.reg_list_size =
6984 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6986 rdev->rlc.cs_data = si_cs_data;
6987 r = sumo_rlc_init(rdev);
6989 DRM_ERROR("Failed to init rlc BOs!\n");
6993 /* allocate wb buffer */
6994 r = radeon_wb_init(rdev);
6998 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7000 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7004 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7006 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7010 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7012 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7016 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7018 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7022 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7024 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7032 if (!rdev->irq.installed) {
7033 r = radeon_irq_kms_init(rdev);
7038 r = si_irq_init(rdev);
7040 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7041 radeon_irq_kms_fini(rdev);
7046 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7047 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7052 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7053 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7058 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7059 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7064 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7065 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7066 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7070 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7071 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7072 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7076 r = si_cp_load_microcode(rdev);
7079 r = si_cp_resume(rdev);
7083 r = cayman_dma_resume(rdev);
7087 si_uvd_resume(rdev);
7088 si_vce_resume(rdev);
7090 r = radeon_ib_pool_init(rdev);
7092 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7096 r = radeon_vm_manager_init(rdev);
7098 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7102 r = radeon_audio_init(rdev);
7109 int si_resume(struct radeon_device *rdev)
7113 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
7114 * posting will perform necessary task to bring back GPU into good
7118 atom_asic_init(rdev->mode_info.atom_context);
7120 /* init golden registers */
7121 si_init_golden_registers(rdev);
7123 if (rdev->pm.pm_method == PM_METHOD_DPM)
7124 radeon_pm_resume(rdev);
7126 rdev->accel_working = true;
7127 r = si_startup(rdev);
7129 DRM_ERROR("si startup failed on resume\n");
7130 rdev->accel_working = false;
7138 int si_suspend(struct radeon_device *rdev)
7140 radeon_pm_suspend(rdev);
7141 radeon_audio_fini(rdev);
7142 radeon_vm_manager_fini(rdev);
7143 si_cp_enable(rdev, false);
7144 cayman_dma_stop(rdev);
7145 if (rdev->has_uvd) {
7146 uvd_v1_0_fini(rdev);
7147 radeon_uvd_suspend(rdev);
7150 radeon_vce_suspend(rdev);
7153 si_irq_suspend(rdev);
7154 radeon_wb_disable(rdev);
7155 si_pcie_gart_disable(rdev);
7159 /* Plan is to move initialization in that function and use
7160 * helper function so that radeon_device_init pretty much
7161 * do nothing more than calling asic specific function. This
7162 * should also allow to remove a bunch of callback function
7165 int si_init(struct radeon_device *rdev)
7167 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7171 if (!radeon_get_bios(rdev)) {
7172 if (ASIC_IS_AVIVO(rdev))
7175 /* Must be an ATOMBIOS */
7176 if (!rdev->is_atom_bios) {
7177 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7180 r = radeon_atombios_init(rdev);
7184 /* Post card if necessary */
7185 if (!radeon_card_posted(rdev)) {
7187 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7190 DRM_INFO("GPU not posted. posting now...\n");
7191 atom_asic_init(rdev->mode_info.atom_context);
7193 /* init golden registers */
7194 si_init_golden_registers(rdev);
7195 /* Initialize scratch registers */
7196 si_scratch_init(rdev);
7197 /* Initialize surface registers */
7198 radeon_surface_init(rdev);
7199 /* Initialize clocks */
7200 radeon_get_clock_info(rdev->ddev);
7203 r = radeon_fence_driver_init(rdev);
7207 /* initialize memory controller */
7208 r = si_mc_init(rdev);
7211 /* Memory manager */
7212 r = radeon_bo_init(rdev);
7216 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7217 !rdev->rlc_fw || !rdev->mc_fw) {
7218 r = si_init_microcode(rdev);
7220 DRM_ERROR("Failed to load firmware!\n");
7225 /* Initialize power management */
7226 radeon_pm_init(rdev);
7228 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7229 ring->ring_obj = NULL;
7230 r600_ring_init(rdev, ring, 1024 * 1024);
7232 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7233 ring->ring_obj = NULL;
7234 r600_ring_init(rdev, ring, 1024 * 1024);
7236 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7237 ring->ring_obj = NULL;
7238 r600_ring_init(rdev, ring, 1024 * 1024);
7240 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7241 ring->ring_obj = NULL;
7242 r600_ring_init(rdev, ring, 64 * 1024);
7244 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7245 ring->ring_obj = NULL;
7246 r600_ring_init(rdev, ring, 64 * 1024);
7251 rdev->ih.ring_obj = NULL;
7252 r600_ih_ring_init(rdev, 64 * 1024);
7254 r = r600_pcie_gart_init(rdev);
7258 rdev->accel_working = true;
7259 r = si_startup(rdev);
7261 dev_err(rdev->dev, "disabling GPU acceleration\n");
7263 cayman_dma_fini(rdev);
7265 sumo_rlc_fini(rdev);
7266 radeon_wb_fini(rdev);
7267 radeon_ib_pool_fini(rdev);
7268 radeon_vm_manager_fini(rdev);
7269 radeon_irq_kms_fini(rdev);
7270 si_pcie_gart_fini(rdev);
7271 rdev->accel_working = false;
7274 /* Don't start up if the MC ucode is missing.
7275 * The default clocks and voltages before the MC ucode
7276 * is loaded are not suffient for advanced operations.
7279 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7286 void si_fini(struct radeon_device *rdev)
7288 radeon_pm_fini(rdev);
7290 cayman_dma_fini(rdev);
7294 sumo_rlc_fini(rdev);
7295 radeon_wb_fini(rdev);
7296 radeon_vm_manager_fini(rdev);
7297 radeon_ib_pool_fini(rdev);
7298 radeon_irq_kms_fini(rdev);
7299 if (rdev->has_uvd) {
7300 uvd_v1_0_fini(rdev);
7301 radeon_uvd_fini(rdev);
7304 radeon_vce_fini(rdev);
7305 si_pcie_gart_fini(rdev);
7306 r600_vram_scratch_fini(rdev);
7307 radeon_gem_fini(rdev);
7308 radeon_fence_driver_fini(rdev);
7309 radeon_bo_fini(rdev);
7310 radeon_atombios_fini(rdev);
7316 * si_get_gpu_clock_counter - return GPU clock counter snapshot
7318 * @rdev: radeon_device pointer
7320 * Fetches a GPU clock counter snapshot (SI).
7321 * Returns the 64 bit clock counter snapshot.
7323 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
7327 mutex_lock(&rdev->gpu_clock_mutex);
7328 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7329 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7330 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7331 mutex_unlock(&rdev->gpu_clock_mutex);
7335 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7337 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
7340 /* bypass vclk and dclk with bclk */
7341 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7342 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7343 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7345 /* put PLL in bypass mode */
7346 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7348 if (!vclk || !dclk) {
7349 /* keep the Bypass mode */
7353 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7354 16384, 0x03FFFFFF, 0, 128, 5,
7355 &fb_div, &vclk_div, &dclk_div);
7359 /* set RESET_ANTI_MUX to 0 */
7360 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7362 /* set VCO_MODE to 1 */
7363 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7365 /* disable sleep mode */
7366 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7368 /* deassert UPLL_RESET */
7369 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7373 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7377 /* assert UPLL_RESET again */
7378 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7380 /* disable spread spectrum. */
7381 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7383 /* set feedback divider */
7384 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7386 /* set ref divider to 0 */
7387 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7389 if (fb_div < 307200)
7390 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7392 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7394 /* set PDIV_A and PDIV_B */
7395 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7396 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7397 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7399 /* give the PLL some time to settle */
7402 /* deassert PLL_RESET */
7403 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7407 /* switch from bypass mode to normal mode */
7408 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7410 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7414 /* switch VCLK and DCLK selection */
7415 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7416 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7417 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7424 static void si_pcie_gen3_enable(struct radeon_device *rdev)
7426 struct pci_dev *root = rdev->pdev->bus->self;
7427 int bridge_pos, gpu_pos;
7428 u32 speed_cntl, mask, current_data_rate;
7432 if (pci_is_root_bus(rdev->pdev->bus))
7435 if (radeon_pcie_gen2 == 0)
7438 if (rdev->flags & RADEON_IS_IGP)
7441 if (!(rdev->flags & RADEON_IS_PCIE))
7444 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7448 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7451 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7452 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7453 LC_CURRENT_DATA_RATE_SHIFT;
7454 if (mask & DRM_PCIE_SPEED_80) {
7455 if (current_data_rate == 2) {
7456 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7459 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7460 } else if (mask & DRM_PCIE_SPEED_50) {
7461 if (current_data_rate == 1) {
7462 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7465 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7468 bridge_pos = pci_pcie_cap(root);
7472 gpu_pos = pci_pcie_cap(rdev->pdev);
7476 if (mask & DRM_PCIE_SPEED_80) {
7477 /* re-try equalization if gen3 is not already enabled */
7478 if (current_data_rate != 2) {
7479 u16 bridge_cfg, gpu_cfg;
7480 u16 bridge_cfg2, gpu_cfg2;
7481 u32 max_lw, current_lw, tmp;
7483 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7484 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7486 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7487 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7489 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7490 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7492 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7493 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7494 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7496 if (current_lw < max_lw) {
7497 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7498 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7499 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7500 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7501 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7502 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7506 for (i = 0; i < 10; i++) {
7508 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7509 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7512 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7513 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7515 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7516 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7518 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7519 tmp |= LC_SET_QUIESCE;
7520 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7522 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7524 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7529 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7530 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7531 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7532 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7534 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7535 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7536 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7537 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7540 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7541 tmp16 &= ~((1 << 4) | (7 << 9));
7542 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7543 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7545 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7546 tmp16 &= ~((1 << 4) | (7 << 9));
7547 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7548 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7550 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7551 tmp &= ~LC_SET_QUIESCE;
7552 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7557 /* set the link speed */
7558 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7559 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7560 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7562 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7564 if (mask & DRM_PCIE_SPEED_80)
7565 tmp16 |= 3; /* gen3 */
7566 else if (mask & DRM_PCIE_SPEED_50)
7567 tmp16 |= 2; /* gen2 */
7569 tmp16 |= 1; /* gen1 */
7570 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7572 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7573 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7574 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7576 for (i = 0; i < rdev->usec_timeout; i++) {
7577 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7578 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7584 static void si_program_aspm(struct radeon_device *rdev)
7587 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7588 bool disable_clkreq = false;
7590 if (radeon_aspm == 0)
7593 if (!(rdev->flags & RADEON_IS_PCIE))
7596 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7597 data &= ~LC_XMIT_N_FTS_MASK;
7598 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7600 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7602 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7603 data |= LC_GO_TO_RECOVERY;
7605 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7607 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7608 data |= P_IGNORE_EDB_ERR;
7610 WREG32_PCIE(PCIE_P_CNTL, data);
7612 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7613 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7614 data |= LC_PMI_TO_L1_DIS;
7616 data |= LC_L0S_INACTIVITY(7);
7619 data |= LC_L1_INACTIVITY(7);
7620 data &= ~LC_PMI_TO_L1_DIS;
7622 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7624 if (!disable_plloff_in_l1) {
7625 bool clk_req_support;
7627 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7628 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7629 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7631 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7633 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7634 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7635 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7637 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7639 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7640 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7641 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7643 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7645 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7646 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7647 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7649 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7651 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7652 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7653 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7655 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7657 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7658 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7660 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7662 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7663 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7665 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7667 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7668 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7670 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7672 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7673 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7675 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7677 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7678 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7680 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7682 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7683 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7685 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7687 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7688 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7690 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7692 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7693 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7694 data |= LC_DYN_LANES_PWR_STATE(3);
7696 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7698 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7699 data &= ~LS2_EXIT_TIME_MASK;
7700 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7701 data |= LS2_EXIT_TIME(5);
7703 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7705 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7706 data &= ~LS2_EXIT_TIME_MASK;
7707 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7708 data |= LS2_EXIT_TIME(5);
7710 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7712 if (!disable_clkreq &&
7713 !pci_is_root_bus(rdev->pdev->bus)) {
7714 struct pci_dev *root = rdev->pdev->bus->self;
7717 clk_req_support = false;
7718 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7719 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7720 clk_req_support = true;
7722 clk_req_support = false;
7725 if (clk_req_support) {
7726 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7727 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7729 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7731 orig = data = RREG32(THM_CLK_CNTL);
7732 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7733 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7735 WREG32(THM_CLK_CNTL, data);
7737 orig = data = RREG32(MISC_CLK_CNTL);
7738 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7739 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7741 WREG32(MISC_CLK_CNTL, data);
7743 orig = data = RREG32(CG_CLKPIN_CNTL);
7744 data &= ~BCLK_AS_XCLK;
7746 WREG32(CG_CLKPIN_CNTL, data);
7748 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7749 data &= ~FORCE_BIF_REFCLK_EN;
7751 WREG32(CG_CLKPIN_CNTL_2, data);
7753 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7754 data &= ~MPLL_CLKOUT_SEL_MASK;
7755 data |= MPLL_CLKOUT_SEL(4);
7757 WREG32(MPLL_BYPASSCLK_SEL, data);
7759 orig = data = RREG32(SPLL_CNTL_MODE);
7760 data &= ~SPLL_REFCLK_SEL_MASK;
7762 WREG32(SPLL_CNTL_MODE, data);
7767 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7770 orig = data = RREG32_PCIE(PCIE_CNTL2);
7771 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7773 WREG32_PCIE(PCIE_CNTL2, data);
7776 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7777 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7778 data = RREG32_PCIE(PCIE_LC_STATUS1);
7779 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7780 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7781 data &= ~LC_L0S_INACTIVITY_MASK;
7783 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7789 int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7793 /* make sure VCEPLL_CTLREQ is deasserted */
7794 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7798 /* assert UPLL_CTLREQ */
7799 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7801 /* wait for CTLACK and CTLACK2 to get asserted */
7802 for (i = 0; i < 100; ++i) {
7803 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7804 if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7809 /* deassert UPLL_CTLREQ */
7810 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7813 DRM_ERROR("Timeout setting UVD clocks!\n");
7820 int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7822 unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7825 /* bypass evclk and ecclk with bclk */
7826 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7827 EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7828 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7830 /* put PLL in bypass mode */
7831 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7832 ~VCEPLL_BYPASS_EN_MASK);
7834 if (!evclk || !ecclk) {
7835 /* keep the Bypass mode, put PLL to sleep */
7836 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7837 ~VCEPLL_SLEEP_MASK);
7841 r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7842 16384, 0x03FFFFFF, 0, 128, 5,
7843 &fb_div, &evclk_div, &ecclk_div);
7847 /* set RESET_ANTI_MUX to 0 */
7848 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7850 /* set VCO_MODE to 1 */
7851 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7852 ~VCEPLL_VCO_MODE_MASK);
7854 /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7855 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7856 ~VCEPLL_SLEEP_MASK);
7857 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7859 /* deassert VCEPLL_RESET */
7860 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7864 r = si_vce_send_vcepll_ctlreq(rdev);
7868 /* assert VCEPLL_RESET again */
7869 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7871 /* disable spread spectrum. */
7872 WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7874 /* set feedback divider */
7875 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7877 /* set ref divider to 0 */
7878 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7880 /* set PDIV_A and PDIV_B */
7881 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7882 VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7883 ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7885 /* give the PLL some time to settle */
7888 /* deassert PLL_RESET */
7889 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7893 /* switch from bypass mode to normal mode */
7894 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7896 r = si_vce_send_vcepll_ctlreq(rdev);
7900 /* switch VCLK and DCLK selection */
7901 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7902 EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7903 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
projects / releases.git / blob