2 * Copyright 2010 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
31 #include <drm/radeon_drm.h>
35 #include "cayman_blit_shaders.h"
36 #include "radeon_ucode.h"
37 #include "clearstate_cayman.h"
40 * Indirect registers accessor
42 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
47 spin_lock_irqsave(&rdev->smc_idx_lock, flags);
48 WREG32(TN_SMC_IND_INDEX_0, (reg));
49 r = RREG32(TN_SMC_IND_DATA_0);
50 spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
54 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
58 spin_lock_irqsave(&rdev->smc_idx_lock, flags);
59 WREG32(TN_SMC_IND_INDEX_0, (reg));
60 WREG32(TN_SMC_IND_DATA_0, (v));
61 spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
64 static const u32 tn_rlc_save_restore_register_list[] =
190 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
191 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
192 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
193 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
194 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
195 extern void evergreen_mc_program(struct radeon_device *rdev);
196 extern void evergreen_irq_suspend(struct radeon_device *rdev);
197 extern int evergreen_mc_init(struct radeon_device *rdev);
198 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
199 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
200 extern void evergreen_program_aspm(struct radeon_device *rdev);
201 extern void sumo_rlc_fini(struct radeon_device *rdev);
202 extern int sumo_rlc_init(struct radeon_device *rdev);
203 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
209 static const u32 cayman_golden_registers2[] =
211 0x3e5c, 0xffffffff, 0x00000000,
212 0x3e48, 0xffffffff, 0x00000000,
213 0x3e4c, 0xffffffff, 0x00000000,
214 0x3e64, 0xffffffff, 0x00000000,
215 0x3e50, 0xffffffff, 0x00000000,
216 0x3e60, 0xffffffff, 0x00000000
219 static const u32 cayman_golden_registers[] =
221 0x5eb4, 0xffffffff, 0x00000002,
222 0x5e78, 0x8f311ff1, 0x001000f0,
223 0x3f90, 0xffff0000, 0xff000000,
224 0x9148, 0xffff0000, 0xff000000,
225 0x3f94, 0xffff0000, 0xff000000,
226 0x914c, 0xffff0000, 0xff000000,
227 0xc78, 0x00000080, 0x00000080,
228 0xbd4, 0x70073777, 0x00011003,
229 0xd02c, 0xbfffff1f, 0x08421000,
230 0xd0b8, 0x73773777, 0x02011003,
231 0x5bc0, 0x00200000, 0x50100000,
232 0x98f8, 0x33773777, 0x02011003,
233 0x98fc, 0xffffffff, 0x76541032,
234 0x7030, 0x31000311, 0x00000011,
235 0x2f48, 0x33773777, 0x42010001,
236 0x6b28, 0x00000010, 0x00000012,
237 0x7728, 0x00000010, 0x00000012,
238 0x10328, 0x00000010, 0x00000012,
239 0x10f28, 0x00000010, 0x00000012,
240 0x11b28, 0x00000010, 0x00000012,
241 0x12728, 0x00000010, 0x00000012,
242 0x240c, 0x000007ff, 0x00000000,
243 0x8a14, 0xf000001f, 0x00000007,
244 0x8b24, 0x3fff3fff, 0x00ff0fff,
245 0x8b10, 0x0000ff0f, 0x00000000,
246 0x28a4c, 0x07ffffff, 0x06000000,
247 0x10c, 0x00000001, 0x00010003,
248 0xa02c, 0xffffffff, 0x0000009b,
249 0x913c, 0x0000010f, 0x01000100,
250 0x8c04, 0xf8ff00ff, 0x40600060,
251 0x28350, 0x00000f01, 0x00000000,
252 0x9508, 0x3700001f, 0x00000002,
253 0x960c, 0xffffffff, 0x54763210,
254 0x88c4, 0x001f3ae3, 0x00000082,
255 0x88d0, 0xffffffff, 0x0f40df40,
256 0x88d4, 0x0000001f, 0x00000010,
257 0x8974, 0xffffffff, 0x00000000
260 static const u32 dvst_golden_registers2[] =
262 0x8f8, 0xffffffff, 0,
263 0x8fc, 0x00380000, 0,
264 0x8f8, 0xffffffff, 1,
268 static const u32 dvst_golden_registers[] =
270 0x690, 0x3fff3fff, 0x20c00033,
271 0x918c, 0x0fff0fff, 0x00010006,
272 0x91a8, 0x0fff0fff, 0x00010006,
273 0x9150, 0xffffdfff, 0x6e944040,
274 0x917c, 0x0fff0fff, 0x00030002,
275 0x9198, 0x0fff0fff, 0x00030002,
276 0x915c, 0x0fff0fff, 0x00010000,
277 0x3f90, 0xffff0001, 0xff000000,
278 0x9178, 0x0fff0fff, 0x00070000,
279 0x9194, 0x0fff0fff, 0x00070000,
280 0x9148, 0xffff0001, 0xff000000,
281 0x9190, 0x0fff0fff, 0x00090008,
282 0x91ac, 0x0fff0fff, 0x00090008,
283 0x3f94, 0xffff0000, 0xff000000,
284 0x914c, 0xffff0000, 0xff000000,
285 0x929c, 0x00000fff, 0x00000001,
286 0x55e4, 0xff607fff, 0xfc000100,
287 0x8a18, 0xff000fff, 0x00000100,
288 0x8b28, 0xff000fff, 0x00000100,
289 0x9144, 0xfffc0fff, 0x00000100,
290 0x6ed8, 0x00010101, 0x00010000,
291 0x9830, 0xffffffff, 0x00000000,
292 0x9834, 0xf00fffff, 0x00000400,
293 0x9838, 0xfffffffe, 0x00000000,
294 0xd0c0, 0xff000fff, 0x00000100,
295 0xd02c, 0xbfffff1f, 0x08421000,
296 0xd0b8, 0x73773777, 0x12010001,
297 0x5bb0, 0x000000f0, 0x00000070,
298 0x98f8, 0x73773777, 0x12010001,
299 0x98fc, 0xffffffff, 0x00000010,
300 0x9b7c, 0x00ff0000, 0x00fc0000,
301 0x8030, 0x00001f0f, 0x0000100a,
302 0x2f48, 0x73773777, 0x12010001,
303 0x2408, 0x00030000, 0x000c007f,
304 0x8a14, 0xf000003f, 0x00000007,
305 0x8b24, 0x3fff3fff, 0x00ff0fff,
306 0x8b10, 0x0000ff0f, 0x00000000,
307 0x28a4c, 0x07ffffff, 0x06000000,
308 0x4d8, 0x00000fff, 0x00000100,
309 0xa008, 0xffffffff, 0x00010000,
310 0x913c, 0xffff03ff, 0x01000100,
311 0x8c00, 0x000000ff, 0x00000003,
312 0x8c04, 0xf8ff00ff, 0x40600060,
313 0x8cf0, 0x1fff1fff, 0x08e00410,
314 0x28350, 0x00000f01, 0x00000000,
315 0x9508, 0xf700071f, 0x00000002,
316 0x960c, 0xffffffff, 0x54763210,
317 0x20ef8, 0x01ff01ff, 0x00000002,
318 0x20e98, 0xfffffbff, 0x00200000,
319 0x2015c, 0xffffffff, 0x00000f40,
320 0x88c4, 0x001f3ae3, 0x00000082,
321 0x8978, 0x3fffffff, 0x04050140,
322 0x88d4, 0x0000001f, 0x00000010,
323 0x8974, 0xffffffff, 0x00000000
326 static const u32 scrapper_golden_registers[] =
328 0x690, 0x3fff3fff, 0x20c00033,
329 0x918c, 0x0fff0fff, 0x00010006,
330 0x918c, 0x0fff0fff, 0x00010006,
331 0x91a8, 0x0fff0fff, 0x00010006,
332 0x91a8, 0x0fff0fff, 0x00010006,
333 0x9150, 0xffffdfff, 0x6e944040,
334 0x9150, 0xffffdfff, 0x6e944040,
335 0x917c, 0x0fff0fff, 0x00030002,
336 0x917c, 0x0fff0fff, 0x00030002,
337 0x9198, 0x0fff0fff, 0x00030002,
338 0x9198, 0x0fff0fff, 0x00030002,
339 0x915c, 0x0fff0fff, 0x00010000,
340 0x915c, 0x0fff0fff, 0x00010000,
341 0x3f90, 0xffff0001, 0xff000000,
342 0x3f90, 0xffff0001, 0xff000000,
343 0x9178, 0x0fff0fff, 0x00070000,
344 0x9178, 0x0fff0fff, 0x00070000,
345 0x9194, 0x0fff0fff, 0x00070000,
346 0x9194, 0x0fff0fff, 0x00070000,
347 0x9148, 0xffff0001, 0xff000000,
348 0x9148, 0xffff0001, 0xff000000,
349 0x9190, 0x0fff0fff, 0x00090008,
350 0x9190, 0x0fff0fff, 0x00090008,
351 0x91ac, 0x0fff0fff, 0x00090008,
352 0x91ac, 0x0fff0fff, 0x00090008,
353 0x3f94, 0xffff0000, 0xff000000,
354 0x3f94, 0xffff0000, 0xff000000,
355 0x914c, 0xffff0000, 0xff000000,
356 0x914c, 0xffff0000, 0xff000000,
357 0x929c, 0x00000fff, 0x00000001,
358 0x929c, 0x00000fff, 0x00000001,
359 0x55e4, 0xff607fff, 0xfc000100,
360 0x8a18, 0xff000fff, 0x00000100,
361 0x8a18, 0xff000fff, 0x00000100,
362 0x8b28, 0xff000fff, 0x00000100,
363 0x8b28, 0xff000fff, 0x00000100,
364 0x9144, 0xfffc0fff, 0x00000100,
365 0x9144, 0xfffc0fff, 0x00000100,
366 0x6ed8, 0x00010101, 0x00010000,
367 0x9830, 0xffffffff, 0x00000000,
368 0x9830, 0xffffffff, 0x00000000,
369 0x9834, 0xf00fffff, 0x00000400,
370 0x9834, 0xf00fffff, 0x00000400,
371 0x9838, 0xfffffffe, 0x00000000,
372 0x9838, 0xfffffffe, 0x00000000,
373 0xd0c0, 0xff000fff, 0x00000100,
374 0xd02c, 0xbfffff1f, 0x08421000,
375 0xd02c, 0xbfffff1f, 0x08421000,
376 0xd0b8, 0x73773777, 0x12010001,
377 0xd0b8, 0x73773777, 0x12010001,
378 0x5bb0, 0x000000f0, 0x00000070,
379 0x98f8, 0x73773777, 0x12010001,
380 0x98f8, 0x73773777, 0x12010001,
381 0x98fc, 0xffffffff, 0x00000010,
382 0x98fc, 0xffffffff, 0x00000010,
383 0x9b7c, 0x00ff0000, 0x00fc0000,
384 0x9b7c, 0x00ff0000, 0x00fc0000,
385 0x8030, 0x00001f0f, 0x0000100a,
386 0x8030, 0x00001f0f, 0x0000100a,
387 0x2f48, 0x73773777, 0x12010001,
388 0x2f48, 0x73773777, 0x12010001,
389 0x2408, 0x00030000, 0x000c007f,
390 0x8a14, 0xf000003f, 0x00000007,
391 0x8a14, 0xf000003f, 0x00000007,
392 0x8b24, 0x3fff3fff, 0x00ff0fff,
393 0x8b24, 0x3fff3fff, 0x00ff0fff,
394 0x8b10, 0x0000ff0f, 0x00000000,
395 0x8b10, 0x0000ff0f, 0x00000000,
396 0x28a4c, 0x07ffffff, 0x06000000,
397 0x28a4c, 0x07ffffff, 0x06000000,
398 0x4d8, 0x00000fff, 0x00000100,
399 0x4d8, 0x00000fff, 0x00000100,
400 0xa008, 0xffffffff, 0x00010000,
401 0xa008, 0xffffffff, 0x00010000,
402 0x913c, 0xffff03ff, 0x01000100,
403 0x913c, 0xffff03ff, 0x01000100,
404 0x90e8, 0x001fffff, 0x010400c0,
405 0x8c00, 0x000000ff, 0x00000003,
406 0x8c00, 0x000000ff, 0x00000003,
407 0x8c04, 0xf8ff00ff, 0x40600060,
408 0x8c04, 0xf8ff00ff, 0x40600060,
409 0x8c30, 0x0000000f, 0x00040005,
410 0x8cf0, 0x1fff1fff, 0x08e00410,
411 0x8cf0, 0x1fff1fff, 0x08e00410,
412 0x900c, 0x00ffffff, 0x0017071f,
413 0x28350, 0x00000f01, 0x00000000,
414 0x28350, 0x00000f01, 0x00000000,
415 0x9508, 0xf700071f, 0x00000002,
416 0x9508, 0xf700071f, 0x00000002,
417 0x9688, 0x00300000, 0x0017000f,
418 0x960c, 0xffffffff, 0x54763210,
419 0x960c, 0xffffffff, 0x54763210,
420 0x20ef8, 0x01ff01ff, 0x00000002,
421 0x20e98, 0xfffffbff, 0x00200000,
422 0x2015c, 0xffffffff, 0x00000f40,
423 0x88c4, 0x001f3ae3, 0x00000082,
424 0x88c4, 0x001f3ae3, 0x00000082,
425 0x8978, 0x3fffffff, 0x04050140,
426 0x8978, 0x3fffffff, 0x04050140,
427 0x88d4, 0x0000001f, 0x00000010,
428 0x88d4, 0x0000001f, 0x00000010,
429 0x8974, 0xffffffff, 0x00000000,
430 0x8974, 0xffffffff, 0x00000000
433 static void ni_init_golden_registers(struct radeon_device *rdev)
435 switch (rdev->family) {
437 radeon_program_register_sequence(rdev,
438 cayman_golden_registers,
439 (const u32)ARRAY_SIZE(cayman_golden_registers));
440 radeon_program_register_sequence(rdev,
441 cayman_golden_registers2,
442 (const u32)ARRAY_SIZE(cayman_golden_registers2));
445 if ((rdev->pdev->device == 0x9900) ||
446 (rdev->pdev->device == 0x9901) ||
447 (rdev->pdev->device == 0x9903) ||
448 (rdev->pdev->device == 0x9904) ||
449 (rdev->pdev->device == 0x9905) ||
450 (rdev->pdev->device == 0x9906) ||
451 (rdev->pdev->device == 0x9907) ||
452 (rdev->pdev->device == 0x9908) ||
453 (rdev->pdev->device == 0x9909) ||
454 (rdev->pdev->device == 0x990A) ||
455 (rdev->pdev->device == 0x990B) ||
456 (rdev->pdev->device == 0x990C) ||
457 (rdev->pdev->device == 0x990D) ||
458 (rdev->pdev->device == 0x990E) ||
459 (rdev->pdev->device == 0x990F) ||
460 (rdev->pdev->device == 0x9910) ||
461 (rdev->pdev->device == 0x9913) ||
462 (rdev->pdev->device == 0x9917) ||
463 (rdev->pdev->device == 0x9918)) {
464 radeon_program_register_sequence(rdev,
465 dvst_golden_registers,
466 (const u32)ARRAY_SIZE(dvst_golden_registers));
467 radeon_program_register_sequence(rdev,
468 dvst_golden_registers2,
469 (const u32)ARRAY_SIZE(dvst_golden_registers2));
471 radeon_program_register_sequence(rdev,
472 scrapper_golden_registers,
473 (const u32)ARRAY_SIZE(scrapper_golden_registers));
474 radeon_program_register_sequence(rdev,
475 dvst_golden_registers2,
476 (const u32)ARRAY_SIZE(dvst_golden_registers2));
484 #define BTC_IO_MC_REGS_SIZE 29
486 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
487 {0x00000077, 0xff010100},
488 {0x00000078, 0x00000000},
489 {0x00000079, 0x00001434},
490 {0x0000007a, 0xcc08ec08},
491 {0x0000007b, 0x00040000},
492 {0x0000007c, 0x000080c0},
493 {0x0000007d, 0x09000000},
494 {0x0000007e, 0x00210404},
495 {0x00000081, 0x08a8e800},
496 {0x00000082, 0x00030444},
497 {0x00000083, 0x00000000},
498 {0x00000085, 0x00000001},
499 {0x00000086, 0x00000002},
500 {0x00000087, 0x48490000},
501 {0x00000088, 0x20244647},
502 {0x00000089, 0x00000005},
503 {0x0000008b, 0x66030000},
504 {0x0000008c, 0x00006603},
505 {0x0000008d, 0x00000100},
506 {0x0000008f, 0x00001c0a},
507 {0x00000090, 0xff000001},
508 {0x00000094, 0x00101101},
509 {0x00000095, 0x00000fff},
510 {0x00000096, 0x00116fff},
511 {0x00000097, 0x60010000},
512 {0x00000098, 0x10010000},
513 {0x00000099, 0x00006000},
514 {0x0000009a, 0x00001000},
515 {0x0000009f, 0x00946a00}
518 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
519 {0x00000077, 0xff010100},
520 {0x00000078, 0x00000000},
521 {0x00000079, 0x00001434},
522 {0x0000007a, 0xcc08ec08},
523 {0x0000007b, 0x00040000},
524 {0x0000007c, 0x000080c0},
525 {0x0000007d, 0x09000000},
526 {0x0000007e, 0x00210404},
527 {0x00000081, 0x08a8e800},
528 {0x00000082, 0x00030444},
529 {0x00000083, 0x00000000},
530 {0x00000085, 0x00000001},
531 {0x00000086, 0x00000002},
532 {0x00000087, 0x48490000},
533 {0x00000088, 0x20244647},
534 {0x00000089, 0x00000005},
535 {0x0000008b, 0x66030000},
536 {0x0000008c, 0x00006603},
537 {0x0000008d, 0x00000100},
538 {0x0000008f, 0x00001c0a},
539 {0x00000090, 0xff000001},
540 {0x00000094, 0x00101101},
541 {0x00000095, 0x00000fff},
542 {0x00000096, 0x00116fff},
543 {0x00000097, 0x60010000},
544 {0x00000098, 0x10010000},
545 {0x00000099, 0x00006000},
546 {0x0000009a, 0x00001000},
547 {0x0000009f, 0x00936a00}
550 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
551 {0x00000077, 0xff010100},
552 {0x00000078, 0x00000000},
553 {0x00000079, 0x00001434},
554 {0x0000007a, 0xcc08ec08},
555 {0x0000007b, 0x00040000},
556 {0x0000007c, 0x000080c0},
557 {0x0000007d, 0x09000000},
558 {0x0000007e, 0x00210404},
559 {0x00000081, 0x08a8e800},
560 {0x00000082, 0x00030444},
561 {0x00000083, 0x00000000},
562 {0x00000085, 0x00000001},
563 {0x00000086, 0x00000002},
564 {0x00000087, 0x48490000},
565 {0x00000088, 0x20244647},
566 {0x00000089, 0x00000005},
567 {0x0000008b, 0x66030000},
568 {0x0000008c, 0x00006603},
569 {0x0000008d, 0x00000100},
570 {0x0000008f, 0x00001c0a},
571 {0x00000090, 0xff000001},
572 {0x00000094, 0x00101101},
573 {0x00000095, 0x00000fff},
574 {0x00000096, 0x00116fff},
575 {0x00000097, 0x60010000},
576 {0x00000098, 0x10010000},
577 {0x00000099, 0x00006000},
578 {0x0000009a, 0x00001000},
579 {0x0000009f, 0x00916a00}
582 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
583 {0x00000077, 0xff010100},
584 {0x00000078, 0x00000000},
585 {0x00000079, 0x00001434},
586 {0x0000007a, 0xcc08ec08},
587 {0x0000007b, 0x00040000},
588 {0x0000007c, 0x000080c0},
589 {0x0000007d, 0x09000000},
590 {0x0000007e, 0x00210404},
591 {0x00000081, 0x08a8e800},
592 {0x00000082, 0x00030444},
593 {0x00000083, 0x00000000},
594 {0x00000085, 0x00000001},
595 {0x00000086, 0x00000002},
596 {0x00000087, 0x48490000},
597 {0x00000088, 0x20244647},
598 {0x00000089, 0x00000005},
599 {0x0000008b, 0x66030000},
600 {0x0000008c, 0x00006603},
601 {0x0000008d, 0x00000100},
602 {0x0000008f, 0x00001c0a},
603 {0x00000090, 0xff000001},
604 {0x00000094, 0x00101101},
605 {0x00000095, 0x00000fff},
606 {0x00000096, 0x00116fff},
607 {0x00000097, 0x60010000},
608 {0x00000098, 0x10010000},
609 {0x00000099, 0x00006000},
610 {0x0000009a, 0x00001000},
611 {0x0000009f, 0x00976b00}
614 int ni_mc_load_microcode(struct radeon_device *rdev)
616 const __be32 *fw_data;
617 u32 mem_type, running, blackout = 0;
619 int i, ucode_size, regs_size;
624 switch (rdev->family) {
626 io_mc_regs = (u32 *)&barts_io_mc_regs;
627 ucode_size = BTC_MC_UCODE_SIZE;
628 regs_size = BTC_IO_MC_REGS_SIZE;
631 io_mc_regs = (u32 *)&turks_io_mc_regs;
632 ucode_size = BTC_MC_UCODE_SIZE;
633 regs_size = BTC_IO_MC_REGS_SIZE;
637 io_mc_regs = (u32 *)&caicos_io_mc_regs;
638 ucode_size = BTC_MC_UCODE_SIZE;
639 regs_size = BTC_IO_MC_REGS_SIZE;
642 io_mc_regs = (u32 *)&cayman_io_mc_regs;
643 ucode_size = CAYMAN_MC_UCODE_SIZE;
644 regs_size = BTC_IO_MC_REGS_SIZE;
648 mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
649 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
651 if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
653 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
654 WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
657 /* reset the engine and set to writable */
658 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
659 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
661 /* load mc io regs */
662 for (i = 0; i < regs_size; i++) {
663 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
664 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
666 /* load the MC ucode */
667 fw_data = (const __be32 *)rdev->mc_fw->data;
668 for (i = 0; i < ucode_size; i++)
669 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
671 /* put the engine back into the active state */
672 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
673 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
674 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
676 /* wait for training to complete */
677 for (i = 0; i < rdev->usec_timeout; i++) {
678 if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
684 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
690 int ni_init_microcode(struct radeon_device *rdev)
692 const char *chip_name;
693 const char *rlc_chip_name;
694 size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
695 size_t smc_req_size = 0;
701 switch (rdev->family) {
704 rlc_chip_name = "BTC";
705 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
706 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
707 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
708 mc_req_size = BTC_MC_UCODE_SIZE * 4;
709 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
713 rlc_chip_name = "BTC";
714 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
715 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
716 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
717 mc_req_size = BTC_MC_UCODE_SIZE * 4;
718 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
721 chip_name = "CAICOS";
722 rlc_chip_name = "BTC";
723 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
724 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
725 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
726 mc_req_size = BTC_MC_UCODE_SIZE * 4;
727 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
730 chip_name = "CAYMAN";
731 rlc_chip_name = "CAYMAN";
732 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
733 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
734 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
735 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
736 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
740 rlc_chip_name = "ARUBA";
741 /* pfp/me same size as CAYMAN */
742 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
743 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
744 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
750 DRM_INFO("Loading %s Microcode\n", chip_name);
752 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
753 err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
756 if (rdev->pfp_fw->size != pfp_req_size) {
757 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
758 rdev->pfp_fw->size, fw_name);
763 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
764 err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
767 if (rdev->me_fw->size != me_req_size) {
768 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
769 rdev->me_fw->size, fw_name);
773 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", rlc_chip_name);
774 err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
777 if (rdev->rlc_fw->size != rlc_req_size) {
778 pr_err("ni_rlc: Bogus length %zu in firmware \"%s\"\n",
779 rdev->rlc_fw->size, fw_name);
783 /* no MC ucode on TN */
784 if (!(rdev->flags & RADEON_IS_IGP)) {
785 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
786 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
789 if (rdev->mc_fw->size != mc_req_size) {
790 pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
791 rdev->mc_fw->size, fw_name);
796 if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
797 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
798 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
800 pr_err("smc: error loading firmware \"%s\"\n", fw_name);
801 release_firmware(rdev->smc_fw);
804 } else if (rdev->smc_fw->size != smc_req_size) {
805 pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
806 rdev->mc_fw->size, fw_name);
814 pr_err("ni_cp: Failed to load firmware \"%s\"\n",
816 release_firmware(rdev->pfp_fw);
818 release_firmware(rdev->me_fw);
820 release_firmware(rdev->rlc_fw);
822 release_firmware(rdev->mc_fw);
829 * cayman_get_allowed_info_register - fetch the register for the info ioctl
831 * @rdev: radeon_device pointer
832 * @reg: register offset in bytes
833 * @val: register value
835 * Returns 0 for success or -EINVAL for an invalid register
838 int cayman_get_allowed_info_register(struct radeon_device *rdev,
843 case GRBM_STATUS_SE0:
844 case GRBM_STATUS_SE1:
847 case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
848 case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
857 int tn_get_temp(struct radeon_device *rdev)
859 u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
860 int actual_temp = (temp / 8) - 49;
862 return actual_temp * 1000;
868 static void cayman_gpu_init(struct radeon_device *rdev)
870 u32 gb_addr_config = 0;
871 u32 mc_shared_chmap, mc_arb_ramcfg;
872 u32 cgts_tcc_disable;
875 u32 cgts_sm_ctrl_reg;
876 u32 hdp_host_path_cntl;
878 u32 disabled_rb_mask;
881 switch (rdev->family) {
883 rdev->config.cayman.max_shader_engines = 2;
884 rdev->config.cayman.max_pipes_per_simd = 4;
885 rdev->config.cayman.max_tile_pipes = 8;
886 rdev->config.cayman.max_simds_per_se = 12;
887 rdev->config.cayman.max_backends_per_se = 4;
888 rdev->config.cayman.max_texture_channel_caches = 8;
889 rdev->config.cayman.max_gprs = 256;
890 rdev->config.cayman.max_threads = 256;
891 rdev->config.cayman.max_gs_threads = 32;
892 rdev->config.cayman.max_stack_entries = 512;
893 rdev->config.cayman.sx_num_of_sets = 8;
894 rdev->config.cayman.sx_max_export_size = 256;
895 rdev->config.cayman.sx_max_export_pos_size = 64;
896 rdev->config.cayman.sx_max_export_smx_size = 192;
897 rdev->config.cayman.max_hw_contexts = 8;
898 rdev->config.cayman.sq_num_cf_insts = 2;
900 rdev->config.cayman.sc_prim_fifo_size = 0x100;
901 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
902 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
903 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
907 rdev->config.cayman.max_shader_engines = 1;
908 rdev->config.cayman.max_pipes_per_simd = 4;
909 rdev->config.cayman.max_tile_pipes = 2;
910 if ((rdev->pdev->device == 0x9900) ||
911 (rdev->pdev->device == 0x9901) ||
912 (rdev->pdev->device == 0x9905) ||
913 (rdev->pdev->device == 0x9906) ||
914 (rdev->pdev->device == 0x9907) ||
915 (rdev->pdev->device == 0x9908) ||
916 (rdev->pdev->device == 0x9909) ||
917 (rdev->pdev->device == 0x990B) ||
918 (rdev->pdev->device == 0x990C) ||
919 (rdev->pdev->device == 0x990F) ||
920 (rdev->pdev->device == 0x9910) ||
921 (rdev->pdev->device == 0x9917) ||
922 (rdev->pdev->device == 0x9999) ||
923 (rdev->pdev->device == 0x999C)) {
924 rdev->config.cayman.max_simds_per_se = 6;
925 rdev->config.cayman.max_backends_per_se = 2;
926 rdev->config.cayman.max_hw_contexts = 8;
927 rdev->config.cayman.sx_max_export_size = 256;
928 rdev->config.cayman.sx_max_export_pos_size = 64;
929 rdev->config.cayman.sx_max_export_smx_size = 192;
930 } else if ((rdev->pdev->device == 0x9903) ||
931 (rdev->pdev->device == 0x9904) ||
932 (rdev->pdev->device == 0x990A) ||
933 (rdev->pdev->device == 0x990D) ||
934 (rdev->pdev->device == 0x990E) ||
935 (rdev->pdev->device == 0x9913) ||
936 (rdev->pdev->device == 0x9918) ||
937 (rdev->pdev->device == 0x999D)) {
938 rdev->config.cayman.max_simds_per_se = 4;
939 rdev->config.cayman.max_backends_per_se = 2;
940 rdev->config.cayman.max_hw_contexts = 8;
941 rdev->config.cayman.sx_max_export_size = 256;
942 rdev->config.cayman.sx_max_export_pos_size = 64;
943 rdev->config.cayman.sx_max_export_smx_size = 192;
944 } else if ((rdev->pdev->device == 0x9919) ||
945 (rdev->pdev->device == 0x9990) ||
946 (rdev->pdev->device == 0x9991) ||
947 (rdev->pdev->device == 0x9994) ||
948 (rdev->pdev->device == 0x9995) ||
949 (rdev->pdev->device == 0x9996) ||
950 (rdev->pdev->device == 0x999A) ||
951 (rdev->pdev->device == 0x99A0)) {
952 rdev->config.cayman.max_simds_per_se = 3;
953 rdev->config.cayman.max_backends_per_se = 1;
954 rdev->config.cayman.max_hw_contexts = 4;
955 rdev->config.cayman.sx_max_export_size = 128;
956 rdev->config.cayman.sx_max_export_pos_size = 32;
957 rdev->config.cayman.sx_max_export_smx_size = 96;
959 rdev->config.cayman.max_simds_per_se = 2;
960 rdev->config.cayman.max_backends_per_se = 1;
961 rdev->config.cayman.max_hw_contexts = 4;
962 rdev->config.cayman.sx_max_export_size = 128;
963 rdev->config.cayman.sx_max_export_pos_size = 32;
964 rdev->config.cayman.sx_max_export_smx_size = 96;
966 rdev->config.cayman.max_texture_channel_caches = 2;
967 rdev->config.cayman.max_gprs = 256;
968 rdev->config.cayman.max_threads = 256;
969 rdev->config.cayman.max_gs_threads = 32;
970 rdev->config.cayman.max_stack_entries = 512;
971 rdev->config.cayman.sx_num_of_sets = 8;
972 rdev->config.cayman.sq_num_cf_insts = 2;
974 rdev->config.cayman.sc_prim_fifo_size = 0x40;
975 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
976 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
977 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
982 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
983 WREG32((0x2c14 + j), 0x00000000);
984 WREG32((0x2c18 + j), 0x00000000);
985 WREG32((0x2c1c + j), 0x00000000);
986 WREG32((0x2c20 + j), 0x00000000);
987 WREG32((0x2c24 + j), 0x00000000);
990 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
991 WREG32(SRBM_INT_CNTL, 0x1);
992 WREG32(SRBM_INT_ACK, 0x1);
994 evergreen_fix_pci_max_read_req_size(rdev);
996 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
997 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
999 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1000 rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1001 if (rdev->config.cayman.mem_row_size_in_kb > 4)
1002 rdev->config.cayman.mem_row_size_in_kb = 4;
1003 /* XXX use MC settings? */
1004 rdev->config.cayman.shader_engine_tile_size = 32;
1005 rdev->config.cayman.num_gpus = 1;
1006 rdev->config.cayman.multi_gpu_tile_size = 64;
1008 tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1009 rdev->config.cayman.num_tile_pipes = (1 << tmp);
1010 tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1011 rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1012 tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1013 rdev->config.cayman.num_shader_engines = tmp + 1;
1014 tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1015 rdev->config.cayman.num_gpus = tmp + 1;
1016 tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1017 rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1018 tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1019 rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1022 /* setup tiling info dword. gb_addr_config is not adequate since it does
1023 * not have bank info, so create a custom tiling dword.
1024 * bits 3:0 num_pipes
1025 * bits 7:4 num_banks
1026 * bits 11:8 group_size
1027 * bits 15:12 row_size
1029 rdev->config.cayman.tile_config = 0;
1030 switch (rdev->config.cayman.num_tile_pipes) {
1033 rdev->config.cayman.tile_config |= (0 << 0);
1036 rdev->config.cayman.tile_config |= (1 << 0);
1039 rdev->config.cayman.tile_config |= (2 << 0);
1042 rdev->config.cayman.tile_config |= (3 << 0);
1046 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1047 if (rdev->flags & RADEON_IS_IGP)
1048 rdev->config.cayman.tile_config |= 1 << 4;
1050 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1051 case 0: /* four banks */
1052 rdev->config.cayman.tile_config |= 0 << 4;
1054 case 1: /* eight banks */
1055 rdev->config.cayman.tile_config |= 1 << 4;
1057 case 2: /* sixteen banks */
1059 rdev->config.cayman.tile_config |= 2 << 4;
1063 rdev->config.cayman.tile_config |=
1064 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1065 rdev->config.cayman.tile_config |=
1066 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1069 for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1070 u32 rb_disable_bitmap;
1072 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1073 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1074 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1076 tmp |= rb_disable_bitmap;
1078 /* enabled rb are just the one not disabled :) */
1079 disabled_rb_mask = tmp;
1081 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1083 /* if all the backends are disabled, fix it up here */
1084 if ((disabled_rb_mask & tmp) == tmp) {
1085 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1086 disabled_rb_mask &= ~(1 << i);
1089 for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1090 u32 simd_disable_bitmap;
1092 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1093 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1094 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1095 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1097 tmp |= simd_disable_bitmap;
1099 rdev->config.cayman.active_simds = hweight32(~tmp);
1101 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1102 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1104 WREG32(GB_ADDR_CONFIG, gb_addr_config);
1105 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1106 if (ASIC_IS_DCE6(rdev))
1107 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1108 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1109 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1110 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1111 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1112 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1113 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1115 if ((rdev->config.cayman.max_backends_per_se == 1) &&
1116 (rdev->flags & RADEON_IS_IGP)) {
1117 if ((disabled_rb_mask & 3) == 2) {
1118 /* RB1 disabled, RB0 enabled */
1121 /* RB0 disabled, RB1 enabled */
1125 tmp = gb_addr_config & NUM_PIPES_MASK;
1126 tmp = r6xx_remap_render_backend(rdev, tmp,
1127 rdev->config.cayman.max_backends_per_se *
1128 rdev->config.cayman.max_shader_engines,
1129 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1131 rdev->config.cayman.backend_map = tmp;
1132 WREG32(GB_BACKEND_MAP, tmp);
1134 cgts_tcc_disable = 0xffff0000;
1135 for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1136 cgts_tcc_disable &= ~(1 << (16 + i));
1137 WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1138 WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1139 WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1140 WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1142 /* reprogram the shader complex */
1143 cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1144 for (i = 0; i < 16; i++)
1145 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1146 WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1148 /* set HW defaults for 3D engine */
1149 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1151 sx_debug_1 = RREG32(SX_DEBUG_1);
1152 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1153 WREG32(SX_DEBUG_1, sx_debug_1);
1155 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1156 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1157 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1158 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1160 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1162 /* need to be explicitly zero-ed */
1163 WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1164 WREG32(SQ_LSTMP_RING_BASE, 0);
1165 WREG32(SQ_HSTMP_RING_BASE, 0);
1166 WREG32(SQ_ESTMP_RING_BASE, 0);
1167 WREG32(SQ_GSTMP_RING_BASE, 0);
1168 WREG32(SQ_VSTMP_RING_BASE, 0);
1169 WREG32(SQ_PSTMP_RING_BASE, 0);
1171 WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1173 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1174 POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1175 SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1177 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1178 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1179 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1182 WREG32(VGT_NUM_INSTANCES, 1);
1184 WREG32(CP_PERFMON_CNTL, 0);
1186 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1187 FETCH_FIFO_HIWATER(0x4) |
1188 DONE_FIFO_HIWATER(0xe0) |
1189 ALU_UPDATE_FIFO_HIWATER(0x8)));
1191 WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1192 WREG32(SQ_CONFIG, (VC_ENABLE |
1197 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1199 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1200 FORCE_EOV_MAX_REZ_CNT(255)));
1202 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1203 AUTO_INVLD_EN(ES_AND_GS_AUTO));
1205 WREG32(VGT_GS_VERTEX_REUSE, 16);
1206 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1208 WREG32(CB_PERF_CTR0_SEL_0, 0);
1209 WREG32(CB_PERF_CTR0_SEL_1, 0);
1210 WREG32(CB_PERF_CTR1_SEL_0, 0);
1211 WREG32(CB_PERF_CTR1_SEL_1, 0);
1212 WREG32(CB_PERF_CTR2_SEL_0, 0);
1213 WREG32(CB_PERF_CTR2_SEL_1, 0);
1214 WREG32(CB_PERF_CTR3_SEL_0, 0);
1215 WREG32(CB_PERF_CTR3_SEL_1, 0);
1217 tmp = RREG32(HDP_MISC_CNTL);
1218 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1219 WREG32(HDP_MISC_CNTL, tmp);
1221 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1222 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1224 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1228 /* set clockgating golden values on TN */
1229 if (rdev->family == CHIP_ARUBA) {
1230 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1232 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1233 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1235 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1242 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1244 /* flush hdp cache */
1245 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1247 /* bits 0-7 are the VM contexts0-7 */
1248 WREG32(VM_INVALIDATE_REQUEST, 1);
1251 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1255 if (rdev->gart.robj == NULL) {
1256 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1259 r = radeon_gart_table_vram_pin(rdev);
1262 /* Setup TLB control */
1263 WREG32(MC_VM_MX_L1_TLB_CNTL,
1266 ENABLE_L1_FRAGMENT_PROCESSING |
1267 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1268 ENABLE_ADVANCED_DRIVER_MODEL |
1269 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1270 /* Setup L2 cache */
1271 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1272 ENABLE_L2_FRAGMENT_PROCESSING |
1273 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1274 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1275 EFFECTIVE_L2_QUEUE_SIZE(7) |
1276 CONTEXT1_IDENTITY_ACCESS_MODE(1));
1277 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1278 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1280 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1281 /* setup context0 */
1282 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1283 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1284 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1285 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1286 (u32)(rdev->dummy_page.addr >> 12));
1287 WREG32(VM_CONTEXT0_CNTL2, 0);
1288 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1289 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1295 /* empty context1-7 */
1296 /* Assign the pt base to something valid for now; the pts used for
1297 * the VMs are determined by the application and setup and assigned
1298 * on the fly in the vm part of radeon_gart.c
1300 for (i = 1; i < 8; i++) {
1301 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1302 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1303 rdev->vm_manager.max_pfn - 1);
1304 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1305 rdev->vm_manager.saved_table_addr[i]);
1308 /* enable context1-7 */
1309 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1310 (u32)(rdev->dummy_page.addr >> 12));
1311 WREG32(VM_CONTEXT1_CNTL2, 4);
1312 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1313 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1314 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1315 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1316 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1317 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1318 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1319 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1320 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1321 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1322 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1323 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1324 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1325 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1327 cayman_pcie_gart_tlb_flush(rdev);
1328 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1329 (unsigned)(rdev->mc.gtt_size >> 20),
1330 (unsigned long long)rdev->gart.table_addr);
1331 rdev->gart.ready = true;
1335 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1339 for (i = 1; i < 8; ++i) {
1340 rdev->vm_manager.saved_table_addr[i] = RREG32(
1341 VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1344 /* Disable all tables */
1345 WREG32(VM_CONTEXT0_CNTL, 0);
1346 WREG32(VM_CONTEXT1_CNTL, 0);
1347 /* Setup TLB control */
1348 WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1349 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1350 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1351 /* Setup L2 cache */
1352 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1353 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1354 EFFECTIVE_L2_QUEUE_SIZE(7) |
1355 CONTEXT1_IDENTITY_ACCESS_MODE(1));
1356 WREG32(VM_L2_CNTL2, 0);
1357 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1358 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1359 radeon_gart_table_vram_unpin(rdev);
1362 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1364 cayman_pcie_gart_disable(rdev);
1365 radeon_gart_table_vram_free(rdev);
1366 radeon_gart_fini(rdev);
1369 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1370 int ring, u32 cp_int_cntl)
1372 WREG32(SRBM_GFX_CNTL, RINGID(ring));
1373 WREG32(CP_INT_CNTL, cp_int_cntl);
1379 void cayman_fence_ring_emit(struct radeon_device *rdev,
1380 struct radeon_fence *fence)
1382 struct radeon_ring *ring = &rdev->ring[fence->ring];
1383 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1384 u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1385 PACKET3_SH_ACTION_ENA;
1387 /* flush read cache over gart for this vmid */
1388 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1389 radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1390 radeon_ring_write(ring, 0xFFFFFFFF);
1391 radeon_ring_write(ring, 0);
1392 radeon_ring_write(ring, 10); /* poll interval */
1393 /* EVENT_WRITE_EOP - flush caches, send int */
1394 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1395 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1396 radeon_ring_write(ring, lower_32_bits(addr));
1397 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1398 radeon_ring_write(ring, fence->seq);
1399 radeon_ring_write(ring, 0);
1402 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1404 struct radeon_ring *ring = &rdev->ring[ib->ring];
1405 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1406 u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1407 PACKET3_SH_ACTION_ENA;
1409 /* set to DX10/11 mode */
1410 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1411 radeon_ring_write(ring, 1);
1413 if (ring->rptr_save_reg) {
1414 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1415 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1416 radeon_ring_write(ring, ((ring->rptr_save_reg -
1417 PACKET3_SET_CONFIG_REG_START) >> 2));
1418 radeon_ring_write(ring, next_rptr);
1421 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1422 radeon_ring_write(ring,
1426 (ib->gpu_addr & 0xFFFFFFFC));
1427 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1428 radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1430 /* flush read cache over gart for this vmid */
1431 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1432 radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1433 radeon_ring_write(ring, 0xFFFFFFFF);
1434 radeon_ring_write(ring, 0);
1435 radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1438 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1441 WREG32(CP_ME_CNTL, 0);
1443 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1444 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1445 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1446 WREG32(SCRATCH_UMSK, 0);
1447 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1451 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1452 struct radeon_ring *ring)
1456 if (rdev->wb.enabled)
1457 rptr = rdev->wb.wb[ring->rptr_offs/4];
1459 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1460 rptr = RREG32(CP_RB0_RPTR);
1461 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1462 rptr = RREG32(CP_RB1_RPTR);
1464 rptr = RREG32(CP_RB2_RPTR);
1470 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1471 struct radeon_ring *ring)
1475 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1476 wptr = RREG32(CP_RB0_WPTR);
1477 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1478 wptr = RREG32(CP_RB1_WPTR);
1480 wptr = RREG32(CP_RB2_WPTR);
1485 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1486 struct radeon_ring *ring)
1488 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1489 WREG32(CP_RB0_WPTR, ring->wptr);
1490 (void)RREG32(CP_RB0_WPTR);
1491 } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1492 WREG32(CP_RB1_WPTR, ring->wptr);
1493 (void)RREG32(CP_RB1_WPTR);
1495 WREG32(CP_RB2_WPTR, ring->wptr);
1496 (void)RREG32(CP_RB2_WPTR);
1500 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1502 const __be32 *fw_data;
1505 if (!rdev->me_fw || !rdev->pfp_fw)
1508 cayman_cp_enable(rdev, false);
1510 fw_data = (const __be32 *)rdev->pfp_fw->data;
1511 WREG32(CP_PFP_UCODE_ADDR, 0);
1512 for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1513 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1514 WREG32(CP_PFP_UCODE_ADDR, 0);
1516 fw_data = (const __be32 *)rdev->me_fw->data;
1517 WREG32(CP_ME_RAM_WADDR, 0);
1518 for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1519 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1521 WREG32(CP_PFP_UCODE_ADDR, 0);
1522 WREG32(CP_ME_RAM_WADDR, 0);
1523 WREG32(CP_ME_RAM_RADDR, 0);
1527 static int cayman_cp_start(struct radeon_device *rdev)
1529 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1532 r = radeon_ring_lock(rdev, ring, 7);
1534 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1537 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1538 radeon_ring_write(ring, 0x1);
1539 radeon_ring_write(ring, 0x0);
1540 radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1541 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1542 radeon_ring_write(ring, 0);
1543 radeon_ring_write(ring, 0);
1544 radeon_ring_unlock_commit(rdev, ring, false);
1546 cayman_cp_enable(rdev, true);
1548 r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1550 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1554 /* setup clear context state */
1555 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1556 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1558 for (i = 0; i < cayman_default_size; i++)
1559 radeon_ring_write(ring, cayman_default_state[i]);
1561 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1562 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1564 /* set clear context state */
1565 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1566 radeon_ring_write(ring, 0);
1568 /* SQ_VTX_BASE_VTX_LOC */
1569 radeon_ring_write(ring, 0xc0026f00);
1570 radeon_ring_write(ring, 0x00000000);
1571 radeon_ring_write(ring, 0x00000000);
1572 radeon_ring_write(ring, 0x00000000);
1575 radeon_ring_write(ring, 0xc0036f00);
1576 radeon_ring_write(ring, 0x00000bc4);
1577 radeon_ring_write(ring, 0xffffffff);
1578 radeon_ring_write(ring, 0xffffffff);
1579 radeon_ring_write(ring, 0xffffffff);
1581 radeon_ring_write(ring, 0xc0026900);
1582 radeon_ring_write(ring, 0x00000316);
1583 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1584 radeon_ring_write(ring, 0x00000010); /* */
1586 radeon_ring_unlock_commit(rdev, ring, false);
1588 /* XXX init other rings */
1593 static void cayman_cp_fini(struct radeon_device *rdev)
1595 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1596 cayman_cp_enable(rdev, false);
1597 radeon_ring_fini(rdev, ring);
1598 radeon_scratch_free(rdev, ring->rptr_save_reg);
1601 static int cayman_cp_resume(struct radeon_device *rdev)
1603 static const int ridx[] = {
1604 RADEON_RING_TYPE_GFX_INDEX,
1605 CAYMAN_RING_TYPE_CP1_INDEX,
1606 CAYMAN_RING_TYPE_CP2_INDEX
1608 static const unsigned cp_rb_cntl[] = {
1613 static const unsigned cp_rb_rptr_addr[] = {
1618 static const unsigned cp_rb_rptr_addr_hi[] = {
1619 CP_RB0_RPTR_ADDR_HI,
1620 CP_RB1_RPTR_ADDR_HI,
1623 static const unsigned cp_rb_base[] = {
1628 static const unsigned cp_rb_rptr[] = {
1633 static const unsigned cp_rb_wptr[] = {
1638 struct radeon_ring *ring;
1641 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1642 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1648 RREG32(GRBM_SOFT_RESET);
1650 WREG32(GRBM_SOFT_RESET, 0);
1651 RREG32(GRBM_SOFT_RESET);
1653 WREG32(CP_SEM_WAIT_TIMER, 0x0);
1654 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1656 /* Set the write pointer delay */
1657 WREG32(CP_RB_WPTR_DELAY, 0);
1659 WREG32(CP_DEBUG, (1 << 27));
1661 /* set the wb address whether it's enabled or not */
1662 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1663 WREG32(SCRATCH_UMSK, 0xff);
1665 for (i = 0; i < 3; ++i) {
1669 /* Set ring buffer size */
1670 ring = &rdev->ring[ridx[i]];
1671 rb_cntl = order_base_2(ring->ring_size / 8);
1672 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1674 rb_cntl |= BUF_SWAP_32BIT;
1676 WREG32(cp_rb_cntl[i], rb_cntl);
1678 /* set the wb address whether it's enabled or not */
1679 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1680 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1681 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1684 /* set the rb base addr, this causes an internal reset of ALL rings */
1685 for (i = 0; i < 3; ++i) {
1686 ring = &rdev->ring[ridx[i]];
1687 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1690 for (i = 0; i < 3; ++i) {
1691 /* Initialize the ring buffer's read and write pointers */
1692 ring = &rdev->ring[ridx[i]];
1693 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1696 WREG32(cp_rb_rptr[i], 0);
1697 WREG32(cp_rb_wptr[i], ring->wptr);
1700 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1703 /* start the rings */
1704 cayman_cp_start(rdev);
1705 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1706 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1707 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1708 /* this only test cp0 */
1709 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1711 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1712 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1713 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1717 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1718 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1723 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1729 tmp = RREG32(GRBM_STATUS);
1730 if (tmp & (PA_BUSY | SC_BUSY |
1732 TA_BUSY | VGT_BUSY |
1734 GDS_BUSY | SPI_BUSY |
1735 IA_BUSY | IA_BUSY_NO_DMA))
1736 reset_mask |= RADEON_RESET_GFX;
1738 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1739 CP_BUSY | CP_COHERENCY_BUSY))
1740 reset_mask |= RADEON_RESET_CP;
1742 if (tmp & GRBM_EE_BUSY)
1743 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1745 /* DMA_STATUS_REG 0 */
1746 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1747 if (!(tmp & DMA_IDLE))
1748 reset_mask |= RADEON_RESET_DMA;
1750 /* DMA_STATUS_REG 1 */
1751 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1752 if (!(tmp & DMA_IDLE))
1753 reset_mask |= RADEON_RESET_DMA1;
1756 tmp = RREG32(SRBM_STATUS2);
1758 reset_mask |= RADEON_RESET_DMA;
1760 if (tmp & DMA1_BUSY)
1761 reset_mask |= RADEON_RESET_DMA1;
1764 tmp = RREG32(SRBM_STATUS);
1765 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1766 reset_mask |= RADEON_RESET_RLC;
1769 reset_mask |= RADEON_RESET_IH;
1772 reset_mask |= RADEON_RESET_SEM;
1774 if (tmp & GRBM_RQ_PENDING)
1775 reset_mask |= RADEON_RESET_GRBM;
1778 reset_mask |= RADEON_RESET_VMC;
1780 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1781 MCC_BUSY | MCD_BUSY))
1782 reset_mask |= RADEON_RESET_MC;
1784 if (evergreen_is_display_hung(rdev))
1785 reset_mask |= RADEON_RESET_DISPLAY;
1788 tmp = RREG32(VM_L2_STATUS);
1790 reset_mask |= RADEON_RESET_VMC;
1792 /* Skip MC reset as it's mostly likely not hung, just busy */
1793 if (reset_mask & RADEON_RESET_MC) {
1794 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1795 reset_mask &= ~RADEON_RESET_MC;
1801 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1803 struct evergreen_mc_save save;
1804 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1807 if (reset_mask == 0)
1810 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1812 evergreen_print_gpu_status_regs(rdev);
1813 dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n",
1815 dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1817 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1819 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1822 /* Disable CP parsing/prefetching */
1823 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1825 if (reset_mask & RADEON_RESET_DMA) {
1827 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1828 tmp &= ~DMA_RB_ENABLE;
1829 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1832 if (reset_mask & RADEON_RESET_DMA1) {
1834 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1835 tmp &= ~DMA_RB_ENABLE;
1836 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1841 evergreen_mc_stop(rdev, &save);
1842 if (evergreen_mc_wait_for_idle(rdev)) {
1843 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1846 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1847 grbm_soft_reset = SOFT_RESET_CB |
1861 if (reset_mask & RADEON_RESET_CP) {
1862 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1864 srbm_soft_reset |= SOFT_RESET_GRBM;
1867 if (reset_mask & RADEON_RESET_DMA)
1868 srbm_soft_reset |= SOFT_RESET_DMA;
1870 if (reset_mask & RADEON_RESET_DMA1)
1871 srbm_soft_reset |= SOFT_RESET_DMA1;
1873 if (reset_mask & RADEON_RESET_DISPLAY)
1874 srbm_soft_reset |= SOFT_RESET_DC;
1876 if (reset_mask & RADEON_RESET_RLC)
1877 srbm_soft_reset |= SOFT_RESET_RLC;
1879 if (reset_mask & RADEON_RESET_SEM)
1880 srbm_soft_reset |= SOFT_RESET_SEM;
1882 if (reset_mask & RADEON_RESET_IH)
1883 srbm_soft_reset |= SOFT_RESET_IH;
1885 if (reset_mask & RADEON_RESET_GRBM)
1886 srbm_soft_reset |= SOFT_RESET_GRBM;
1888 if (reset_mask & RADEON_RESET_VMC)
1889 srbm_soft_reset |= SOFT_RESET_VMC;
1891 if (!(rdev->flags & RADEON_IS_IGP)) {
1892 if (reset_mask & RADEON_RESET_MC)
1893 srbm_soft_reset |= SOFT_RESET_MC;
1896 if (grbm_soft_reset) {
1897 tmp = RREG32(GRBM_SOFT_RESET);
1898 tmp |= grbm_soft_reset;
1899 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1900 WREG32(GRBM_SOFT_RESET, tmp);
1901 tmp = RREG32(GRBM_SOFT_RESET);
1905 tmp &= ~grbm_soft_reset;
1906 WREG32(GRBM_SOFT_RESET, tmp);
1907 tmp = RREG32(GRBM_SOFT_RESET);
1910 if (srbm_soft_reset) {
1911 tmp = RREG32(SRBM_SOFT_RESET);
1912 tmp |= srbm_soft_reset;
1913 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1914 WREG32(SRBM_SOFT_RESET, tmp);
1915 tmp = RREG32(SRBM_SOFT_RESET);
1919 tmp &= ~srbm_soft_reset;
1920 WREG32(SRBM_SOFT_RESET, tmp);
1921 tmp = RREG32(SRBM_SOFT_RESET);
1924 /* Wait a little for things to settle down */
1927 evergreen_mc_resume(rdev, &save);
1930 evergreen_print_gpu_status_regs(rdev);
1933 int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1938 evergreen_gpu_pci_config_reset(rdev);
1942 reset_mask = cayman_gpu_check_soft_reset(rdev);
1945 r600_set_bios_scratch_engine_hung(rdev, true);
1947 cayman_gpu_soft_reset(rdev, reset_mask);
1949 reset_mask = cayman_gpu_check_soft_reset(rdev);
1952 evergreen_gpu_pci_config_reset(rdev);
1954 r600_set_bios_scratch_engine_hung(rdev, false);
1960 * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1962 * @rdev: radeon_device pointer
1963 * @ring: radeon_ring structure holding ring information
1965 * Check if the GFX engine is locked up.
1966 * Returns true if the engine appears to be locked up, false if not.
1968 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1970 u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1972 if (!(reset_mask & (RADEON_RESET_GFX |
1973 RADEON_RESET_COMPUTE |
1974 RADEON_RESET_CP))) {
1975 radeon_ring_lockup_update(rdev, ring);
1978 return radeon_ring_test_lockup(rdev, ring);
1981 static void cayman_uvd_init(struct radeon_device *rdev)
1988 r = radeon_uvd_init(rdev);
1990 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1992 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1993 * to early fails uvd_v2_2_resume() and thus nothing happens
1994 * there. So it is pointless to try to go through that code
1995 * hence why we disable uvd here.
2000 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
2001 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
2004 static void cayman_uvd_start(struct radeon_device *rdev)
2011 r = uvd_v2_2_resume(rdev);
2013 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2016 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2018 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2024 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2027 static void cayman_uvd_resume(struct radeon_device *rdev)
2029 struct radeon_ring *ring;
2032 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2035 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2036 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2038 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2041 r = uvd_v1_0_init(rdev);
2043 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2048 static void cayman_vce_init(struct radeon_device *rdev)
2052 /* Only set for CHIP_ARUBA */
2056 r = radeon_vce_init(rdev);
2058 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2060 * At this point rdev->vce.vcpu_bo is NULL which trickles down
2061 * to early fails cayman_vce_start() and thus nothing happens
2062 * there. So it is pointless to try to go through that code
2063 * hence why we disable vce here.
2068 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2069 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2070 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2071 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2074 static void cayman_vce_start(struct radeon_device *rdev)
2081 r = radeon_vce_resume(rdev);
2083 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2086 r = vce_v1_0_resume(rdev);
2088 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2091 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2093 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2096 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2098 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2104 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2105 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2108 static void cayman_vce_resume(struct radeon_device *rdev)
2110 struct radeon_ring *ring;
2113 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2116 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2117 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2119 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2122 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2123 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2125 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2128 r = vce_v1_0_init(rdev);
2130 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2135 static int cayman_startup(struct radeon_device *rdev)
2137 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2140 /* enable pcie gen2 link */
2141 evergreen_pcie_gen2_enable(rdev);
2143 evergreen_program_aspm(rdev);
2145 /* scratch needs to be initialized before MC */
2146 r = r600_vram_scratch_init(rdev);
2150 evergreen_mc_program(rdev);
2152 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2153 r = ni_mc_load_microcode(rdev);
2155 DRM_ERROR("Failed to load MC firmware!\n");
2160 r = cayman_pcie_gart_enable(rdev);
2163 cayman_gpu_init(rdev);
2165 /* allocate rlc buffers */
2166 if (rdev->flags & RADEON_IS_IGP) {
2167 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2168 rdev->rlc.reg_list_size =
2169 (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2170 rdev->rlc.cs_data = cayman_cs_data;
2171 r = sumo_rlc_init(rdev);
2173 DRM_ERROR("Failed to init rlc BOs!\n");
2178 /* allocate wb buffer */
2179 r = radeon_wb_init(rdev);
2183 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2185 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2189 cayman_uvd_start(rdev);
2190 cayman_vce_start(rdev);
2192 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2194 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2198 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2200 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2204 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2206 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2210 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2212 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2217 if (!rdev->irq.installed) {
2218 r = radeon_irq_kms_init(rdev);
2223 r = r600_irq_init(rdev);
2225 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2226 radeon_irq_kms_fini(rdev);
2229 evergreen_irq_set(rdev);
2231 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2236 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2237 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2238 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2242 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2243 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2244 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2248 r = cayman_cp_load_microcode(rdev);
2251 r = cayman_cp_resume(rdev);
2255 r = cayman_dma_resume(rdev);
2259 cayman_uvd_resume(rdev);
2260 cayman_vce_resume(rdev);
2262 r = radeon_ib_pool_init(rdev);
2264 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2268 r = radeon_vm_manager_init(rdev);
2270 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2274 r = radeon_audio_init(rdev);
2281 int cayman_resume(struct radeon_device *rdev)
2285 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2286 * posting will perform necessary task to bring back GPU into good
2290 atom_asic_init(rdev->mode_info.atom_context);
2292 /* init golden registers */
2293 ni_init_golden_registers(rdev);
2295 if (rdev->pm.pm_method == PM_METHOD_DPM)
2296 radeon_pm_resume(rdev);
2298 rdev->accel_working = true;
2299 r = cayman_startup(rdev);
2301 DRM_ERROR("cayman startup failed on resume\n");
2302 rdev->accel_working = false;
2308 int cayman_suspend(struct radeon_device *rdev)
2310 radeon_pm_suspend(rdev);
2311 radeon_audio_fini(rdev);
2312 radeon_vm_manager_fini(rdev);
2313 cayman_cp_enable(rdev, false);
2314 cayman_dma_stop(rdev);
2315 if (rdev->has_uvd) {
2316 uvd_v1_0_fini(rdev);
2317 radeon_uvd_suspend(rdev);
2319 evergreen_irq_suspend(rdev);
2320 radeon_wb_disable(rdev);
2321 cayman_pcie_gart_disable(rdev);
2325 /* Plan is to move initialization in that function and use
2326 * helper function so that radeon_device_init pretty much
2327 * do nothing more than calling asic specific function. This
2328 * should also allow to remove a bunch of callback function
2331 int cayman_init(struct radeon_device *rdev)
2333 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2337 if (!radeon_get_bios(rdev)) {
2338 if (ASIC_IS_AVIVO(rdev))
2341 /* Must be an ATOMBIOS */
2342 if (!rdev->is_atom_bios) {
2343 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2346 r = radeon_atombios_init(rdev);
2350 /* Post card if necessary */
2351 if (!radeon_card_posted(rdev)) {
2353 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2356 DRM_INFO("GPU not posted. posting now...\n");
2357 atom_asic_init(rdev->mode_info.atom_context);
2359 /* init golden registers */
2360 ni_init_golden_registers(rdev);
2361 /* Initialize scratch registers */
2362 r600_scratch_init(rdev);
2363 /* Initialize surface registers */
2364 radeon_surface_init(rdev);
2365 /* Initialize clocks */
2366 radeon_get_clock_info(rdev->ddev);
2368 r = radeon_fence_driver_init(rdev);
2371 /* initialize memory controller */
2372 r = evergreen_mc_init(rdev);
2375 /* Memory manager */
2376 r = radeon_bo_init(rdev);
2380 if (rdev->flags & RADEON_IS_IGP) {
2381 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2382 r = ni_init_microcode(rdev);
2384 DRM_ERROR("Failed to load firmware!\n");
2389 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2390 r = ni_init_microcode(rdev);
2392 DRM_ERROR("Failed to load firmware!\n");
2398 /* Initialize power management */
2399 radeon_pm_init(rdev);
2401 ring->ring_obj = NULL;
2402 r600_ring_init(rdev, ring, 1024 * 1024);
2404 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2405 ring->ring_obj = NULL;
2406 r600_ring_init(rdev, ring, 64 * 1024);
2408 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2409 ring->ring_obj = NULL;
2410 r600_ring_init(rdev, ring, 64 * 1024);
2412 cayman_uvd_init(rdev);
2413 cayman_vce_init(rdev);
2415 rdev->ih.ring_obj = NULL;
2416 r600_ih_ring_init(rdev, 64 * 1024);
2418 r = r600_pcie_gart_init(rdev);
2422 rdev->accel_working = true;
2423 r = cayman_startup(rdev);
2425 dev_err(rdev->dev, "disabling GPU acceleration\n");
2426 cayman_cp_fini(rdev);
2427 cayman_dma_fini(rdev);
2428 r600_irq_fini(rdev);
2429 if (rdev->flags & RADEON_IS_IGP)
2430 sumo_rlc_fini(rdev);
2431 radeon_wb_fini(rdev);
2432 radeon_ib_pool_fini(rdev);
2433 radeon_vm_manager_fini(rdev);
2434 radeon_irq_kms_fini(rdev);
2435 cayman_pcie_gart_fini(rdev);
2436 rdev->accel_working = false;
2439 /* Don't start up if the MC ucode is missing.
2440 * The default clocks and voltages before the MC ucode
2441 * is loaded are not suffient for advanced operations.
2443 * We can skip this check for TN, because there is no MC
2446 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2447 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2454 void cayman_fini(struct radeon_device *rdev)
2456 radeon_pm_fini(rdev);
2457 cayman_cp_fini(rdev);
2458 cayman_dma_fini(rdev);
2459 r600_irq_fini(rdev);
2460 if (rdev->flags & RADEON_IS_IGP)
2461 sumo_rlc_fini(rdev);
2462 radeon_wb_fini(rdev);
2463 radeon_vm_manager_fini(rdev);
2464 radeon_ib_pool_fini(rdev);
2465 radeon_irq_kms_fini(rdev);
2466 uvd_v1_0_fini(rdev);
2467 radeon_uvd_fini(rdev);
2469 radeon_vce_fini(rdev);
2470 cayman_pcie_gart_fini(rdev);
2471 r600_vram_scratch_fini(rdev);
2472 radeon_gem_fini(rdev);
2473 radeon_fence_driver_fini(rdev);
2474 radeon_bo_fini(rdev);
2475 radeon_atombios_fini(rdev);
2483 int cayman_vm_init(struct radeon_device *rdev)
2486 rdev->vm_manager.nvm = 8;
2487 /* base offset of vram pages */
2488 if (rdev->flags & RADEON_IS_IGP) {
2489 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2491 rdev->vm_manager.vram_base_offset = tmp;
2493 rdev->vm_manager.vram_base_offset = 0;
2497 void cayman_vm_fini(struct radeon_device *rdev)
2502 * cayman_vm_decode_fault - print human readable fault info
2504 * @rdev: radeon_device pointer
2505 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2506 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2508 * Print human readable fault information (cayman/TN).
2510 void cayman_vm_decode_fault(struct radeon_device *rdev,
2511 u32 status, u32 addr)
2513 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2514 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2515 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2607 block = "TC_TFETCH";
2617 block = "TC_VFETCH";
2656 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2657 protections, vmid, addr,
2658 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2663 * cayman_vm_flush - vm flush using the CP
2665 * @rdev: radeon_device pointer
2667 * Update the page table base and flush the VM TLB
2668 * using the CP (cayman-si).
2670 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2671 unsigned vm_id, uint64_t pd_addr)
2673 radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2674 radeon_ring_write(ring, pd_addr >> 12);
2676 /* flush hdp cache */
2677 radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2678 radeon_ring_write(ring, 0x1);
2680 /* bits 0-7 are the VM contexts0-7 */
2681 radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2682 radeon_ring_write(ring, 1 << vm_id);
2684 /* wait for the invalidate to complete */
2685 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2686 radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */
2687 WAIT_REG_MEM_ENGINE(0))); /* me */
2688 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2689 radeon_ring_write(ring, 0);
2690 radeon_ring_write(ring, 0); /* ref */
2691 radeon_ring_write(ring, 0); /* mask */
2692 radeon_ring_write(ring, 0x20); /* poll interval */
2694 /* sync PFP to ME, otherwise we might get invalid PFP reads */
2695 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2696 radeon_ring_write(ring, 0x0);
2699 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2701 struct atom_clock_dividers dividers;
2704 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2705 ecclk, false, ÷rs);
2709 for (i = 0; i < 100; i++) {
2710 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2717 WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2719 for (i = 0; i < 100; i++) {
2720 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)