drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include <linux/slab.h>
  29
  30 #include <drm/radeon_drm.h>
  31
  32 #include "atom.h"
  33 #include "cayman_blit_shaders.h"
  34 #include "clearstate_cayman.h"
  35 #include "evergreen.h"
  36 #include "ni.h"
  37 #include "ni_reg.h"
  38 #include "nid.h"
  39 #include "radeon.h"
  40 #include "radeon_asic.h"
  41 #include "radeon_audio.h"
  42 #include "radeon_ucode.h"
  43
  44 /*
  45  * Indirect registers accessor
  46  */
  47 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  48 {
  49         unsigned long flags;
  50         u32 r;
  51
  52         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  53         WREG32(TN_SMC_IND_INDEX_0, (reg));
  54         r = RREG32(TN_SMC_IND_DATA_0);
  55         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  56         return r;
  57 }
  58
  59 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  60 {
  61         unsigned long flags;
  62
  63         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  64         WREG32(TN_SMC_IND_INDEX_0, (reg));
  65         WREG32(TN_SMC_IND_DATA_0, (v));
  66         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  67 }
  68
  69 static const u32 tn_rlc_save_restore_register_list[] =
  70 {
  71         0x98fc,
  72         0x98f0,
  73         0x9834,
  74         0x9838,
  75         0x9870,
  76         0x9874,
  77         0x8a14,
  78         0x8b24,
  79         0x8bcc,
  80         0x8b10,
  81         0x8c30,
  82         0x8d00,
  83         0x8d04,
  84         0x8c00,
  85         0x8c04,
  86         0x8c10,
  87         0x8c14,
  88         0x8d8c,
  89         0x8cf0,
  90         0x8e38,
  91         0x9508,
  92         0x9688,
  93         0x9608,
  94         0x960c,
  95         0x9610,
  96         0x9614,
  97         0x88c4,
  98         0x8978,
  99         0x88d4,
 100         0x900c,
 101         0x9100,
 102         0x913c,
 103         0x90e8,
 104         0x9354,
 105         0xa008,
 106         0x98f8,
 107         0x9148,
 108         0x914c,
 109         0x3f94,
 110         0x98f4,
 111         0x9b7c,
 112         0x3f8c,
 113         0x8950,
 114         0x8954,
 115         0x8a18,
 116         0x8b28,
 117         0x9144,
 118         0x3f90,
 119         0x915c,
 120         0x9160,
 121         0x9178,
 122         0x917c,
 123         0x9180,
 124         0x918c,
 125         0x9190,
 126         0x9194,
 127         0x9198,
 128         0x919c,
 129         0x91a8,
 130         0x91ac,
 131         0x91b0,
 132         0x91b4,
 133         0x91b8,
 134         0x91c4,
 135         0x91c8,
 136         0x91cc,
 137         0x91d0,
 138         0x91d4,
 139         0x91e0,
 140         0x91e4,
 141         0x91ec,
 142         0x91f0,
 143         0x91f4,
 144         0x9200,
 145         0x9204,
 146         0x929c,
 147         0x8030,
 148         0x9150,
 149         0x9a60,
 150         0x920c,
 151         0x9210,
 152         0x9228,
 153         0x922c,
 154         0x9244,
 155         0x9248,
 156         0x91e8,
 157         0x9294,
 158         0x9208,
 159         0x9224,
 160         0x9240,
 161         0x9220,
 162         0x923c,
 163         0x9258,
 164         0x9744,
 165         0xa200,
 166         0xa204,
 167         0xa208,
 168         0xa20c,
 169         0x8d58,
 170         0x9030,
 171         0x9034,
 172         0x9038,
 173         0x903c,
 174         0x9040,
 175         0x9654,
 176         0x897c,
 177         0xa210,
 178         0xa214,
 179         0x9868,
 180         0xa02c,
 181         0x9664,
 182         0x9698,
 183         0x949c,
 184         0x8e10,
 185         0x8e18,
 186         0x8c50,
 187         0x8c58,
 188         0x8c60,
 189         0x8c68,
 190         0x89b4,
 191         0x9830,
 192         0x802c,
 193 };
 194
 195 /* Firmware Names */
 196 /*(DEBLOBBED)*/
 197
 198
 199 static const u32 cayman_golden_registers2[] =
 200 {
 201         0x3e5c, 0xffffffff, 0x00000000,
 202         0x3e48, 0xffffffff, 0x00000000,
 203         0x3e4c, 0xffffffff, 0x00000000,
 204         0x3e64, 0xffffffff, 0x00000000,
 205         0x3e50, 0xffffffff, 0x00000000,
 206         0x3e60, 0xffffffff, 0x00000000
 207 };
 208
 209 static const u32 cayman_golden_registers[] =
 210 {
 211         0x5eb4, 0xffffffff, 0x00000002,
 212         0x5e78, 0x8f311ff1, 0x001000f0,
 213         0x3f90, 0xffff0000, 0xff000000,
 214         0x9148, 0xffff0000, 0xff000000,
 215         0x3f94, 0xffff0000, 0xff000000,
 216         0x914c, 0xffff0000, 0xff000000,
 217         0xc78, 0x00000080, 0x00000080,
 218         0xbd4, 0x70073777, 0x00011003,
 219         0xd02c, 0xbfffff1f, 0x08421000,
 220         0xd0b8, 0x73773777, 0x02011003,
 221         0x5bc0, 0x00200000, 0x50100000,
 222         0x98f8, 0x33773777, 0x02011003,
 223         0x98fc, 0xffffffff, 0x76541032,
 224         0x7030, 0x31000311, 0x00000011,
 225         0x2f48, 0x33773777, 0x42010001,
 226         0x6b28, 0x00000010, 0x00000012,
 227         0x7728, 0x00000010, 0x00000012,
 228         0x10328, 0x00000010, 0x00000012,
 229         0x10f28, 0x00000010, 0x00000012,
 230         0x11b28, 0x00000010, 0x00000012,
 231         0x12728, 0x00000010, 0x00000012,
 232         0x240c, 0x000007ff, 0x00000000,
 233         0x8a14, 0xf000001f, 0x00000007,
 234         0x8b24, 0x3fff3fff, 0x00ff0fff,
 235         0x8b10, 0x0000ff0f, 0x00000000,
 236         0x28a4c, 0x07ffffff, 0x06000000,
 237         0x10c, 0x00000001, 0x00010003,
 238         0xa02c, 0xffffffff, 0x0000009b,
 239         0x913c, 0x0000010f, 0x01000100,
 240         0x8c04, 0xf8ff00ff, 0x40600060,
 241         0x28350, 0x00000f01, 0x00000000,
 242         0x9508, 0x3700001f, 0x00000002,
 243         0x960c, 0xffffffff, 0x54763210,
 244         0x88c4, 0x001f3ae3, 0x00000082,
 245         0x88d0, 0xffffffff, 0x0f40df40,
 246         0x88d4, 0x0000001f, 0x00000010,
 247         0x8974, 0xffffffff, 0x00000000
 248 };
 249
 250 static const u32 dvst_golden_registers2[] =
 251 {
 252         0x8f8, 0xffffffff, 0,
 253         0x8fc, 0x00380000, 0,
 254         0x8f8, 0xffffffff, 1,
 255         0x8fc, 0x0e000000, 0
 256 };
 257
 258 static const u32 dvst_golden_registers[] =
 259 {
 260         0x690, 0x3fff3fff, 0x20c00033,
 261         0x918c, 0x0fff0fff, 0x00010006,
 262         0x91a8, 0x0fff0fff, 0x00010006,
 263         0x9150, 0xffffdfff, 0x6e944040,
 264         0x917c, 0x0fff0fff, 0x00030002,
 265         0x9198, 0x0fff0fff, 0x00030002,
 266         0x915c, 0x0fff0fff, 0x00010000,
 267         0x3f90, 0xffff0001, 0xff000000,
 268         0x9178, 0x0fff0fff, 0x00070000,
 269         0x9194, 0x0fff0fff, 0x00070000,
 270         0x9148, 0xffff0001, 0xff000000,
 271         0x9190, 0x0fff0fff, 0x00090008,
 272         0x91ac, 0x0fff0fff, 0x00090008,
 273         0x3f94, 0xffff0000, 0xff000000,
 274         0x914c, 0xffff0000, 0xff000000,
 275         0x929c, 0x00000fff, 0x00000001,
 276         0x55e4, 0xff607fff, 0xfc000100,
 277         0x8a18, 0xff000fff, 0x00000100,
 278         0x8b28, 0xff000fff, 0x00000100,
 279         0x9144, 0xfffc0fff, 0x00000100,
 280         0x6ed8, 0x00010101, 0x00010000,
 281         0x9830, 0xffffffff, 0x00000000,
 282         0x9834, 0xf00fffff, 0x00000400,
 283         0x9838, 0xfffffffe, 0x00000000,
 284         0xd0c0, 0xff000fff, 0x00000100,
 285         0xd02c, 0xbfffff1f, 0x08421000,
 286         0xd0b8, 0x73773777, 0x12010001,
 287         0x5bb0, 0x000000f0, 0x00000070,
 288         0x98f8, 0x73773777, 0x12010001,
 289         0x98fc, 0xffffffff, 0x00000010,
 290         0x9b7c, 0x00ff0000, 0x00fc0000,
 291         0x8030, 0x00001f0f, 0x0000100a,
 292         0x2f48, 0x73773777, 0x12010001,
 293         0x2408, 0x00030000, 0x000c007f,
 294         0x8a14, 0xf000003f, 0x00000007,
 295         0x8b24, 0x3fff3fff, 0x00ff0fff,
 296         0x8b10, 0x0000ff0f, 0x00000000,
 297         0x28a4c, 0x07ffffff, 0x06000000,
 298         0x4d8, 0x00000fff, 0x00000100,
 299         0xa008, 0xffffffff, 0x00010000,
 300         0x913c, 0xffff03ff, 0x01000100,
 301         0x8c00, 0x000000ff, 0x00000003,
 302         0x8c04, 0xf8ff00ff, 0x40600060,
 303         0x8cf0, 0x1fff1fff, 0x08e00410,
 304         0x28350, 0x00000f01, 0x00000000,
 305         0x9508, 0xf700071f, 0x00000002,
 306         0x960c, 0xffffffff, 0x54763210,
 307         0x20ef8, 0x01ff01ff, 0x00000002,
 308         0x20e98, 0xfffffbff, 0x00200000,
 309         0x2015c, 0xffffffff, 0x00000f40,
 310         0x88c4, 0x001f3ae3, 0x00000082,
 311         0x8978, 0x3fffffff, 0x04050140,
 312         0x88d4, 0x0000001f, 0x00000010,
 313         0x8974, 0xffffffff, 0x00000000
 314 };
 315
 316 static const u32 scrapper_golden_registers[] =
 317 {
 318         0x690, 0x3fff3fff, 0x20c00033,
 319         0x918c, 0x0fff0fff, 0x00010006,
 320         0x918c, 0x0fff0fff, 0x00010006,
 321         0x91a8, 0x0fff0fff, 0x00010006,
 322         0x91a8, 0x0fff0fff, 0x00010006,
 323         0x9150, 0xffffdfff, 0x6e944040,
 324         0x9150, 0xffffdfff, 0x6e944040,
 325         0x917c, 0x0fff0fff, 0x00030002,
 326         0x917c, 0x0fff0fff, 0x00030002,
 327         0x9198, 0x0fff0fff, 0x00030002,
 328         0x9198, 0x0fff0fff, 0x00030002,
 329         0x915c, 0x0fff0fff, 0x00010000,
 330         0x915c, 0x0fff0fff, 0x00010000,
 331         0x3f90, 0xffff0001, 0xff000000,
 332         0x3f90, 0xffff0001, 0xff000000,
 333         0x9178, 0x0fff0fff, 0x00070000,
 334         0x9178, 0x0fff0fff, 0x00070000,
 335         0x9194, 0x0fff0fff, 0x00070000,
 336         0x9194, 0x0fff0fff, 0x00070000,
 337         0x9148, 0xffff0001, 0xff000000,
 338         0x9148, 0xffff0001, 0xff000000,
 339         0x9190, 0x0fff0fff, 0x00090008,
 340         0x9190, 0x0fff0fff, 0x00090008,
 341         0x91ac, 0x0fff0fff, 0x00090008,
 342         0x91ac, 0x0fff0fff, 0x00090008,
 343         0x3f94, 0xffff0000, 0xff000000,
 344         0x3f94, 0xffff0000, 0xff000000,
 345         0x914c, 0xffff0000, 0xff000000,
 346         0x914c, 0xffff0000, 0xff000000,
 347         0x929c, 0x00000fff, 0x00000001,
 348         0x929c, 0x00000fff, 0x00000001,
 349         0x55e4, 0xff607fff, 0xfc000100,
 350         0x8a18, 0xff000fff, 0x00000100,
 351         0x8a18, 0xff000fff, 0x00000100,
 352         0x8b28, 0xff000fff, 0x00000100,
 353         0x8b28, 0xff000fff, 0x00000100,
 354         0x9144, 0xfffc0fff, 0x00000100,
 355         0x9144, 0xfffc0fff, 0x00000100,
 356         0x6ed8, 0x00010101, 0x00010000,
 357         0x9830, 0xffffffff, 0x00000000,
 358         0x9830, 0xffffffff, 0x00000000,
 359         0x9834, 0xf00fffff, 0x00000400,
 360         0x9834, 0xf00fffff, 0x00000400,
 361         0x9838, 0xfffffffe, 0x00000000,
 362         0x9838, 0xfffffffe, 0x00000000,
 363         0xd0c0, 0xff000fff, 0x00000100,
 364         0xd02c, 0xbfffff1f, 0x08421000,
 365         0xd02c, 0xbfffff1f, 0x08421000,
 366         0xd0b8, 0x73773777, 0x12010001,
 367         0xd0b8, 0x73773777, 0x12010001,
 368         0x5bb0, 0x000000f0, 0x00000070,
 369         0x98f8, 0x73773777, 0x12010001,
 370         0x98f8, 0x73773777, 0x12010001,
 371         0x98fc, 0xffffffff, 0x00000010,
 372         0x98fc, 0xffffffff, 0x00000010,
 373         0x9b7c, 0x00ff0000, 0x00fc0000,
 374         0x9b7c, 0x00ff0000, 0x00fc0000,
 375         0x8030, 0x00001f0f, 0x0000100a,
 376         0x8030, 0x00001f0f, 0x0000100a,
 377         0x2f48, 0x73773777, 0x12010001,
 378         0x2f48, 0x73773777, 0x12010001,
 379         0x2408, 0x00030000, 0x000c007f,
 380         0x8a14, 0xf000003f, 0x00000007,
 381         0x8a14, 0xf000003f, 0x00000007,
 382         0x8b24, 0x3fff3fff, 0x00ff0fff,
 383         0x8b24, 0x3fff3fff, 0x00ff0fff,
 384         0x8b10, 0x0000ff0f, 0x00000000,
 385         0x8b10, 0x0000ff0f, 0x00000000,
 386         0x28a4c, 0x07ffffff, 0x06000000,
 387         0x28a4c, 0x07ffffff, 0x06000000,
 388         0x4d8, 0x00000fff, 0x00000100,
 389         0x4d8, 0x00000fff, 0x00000100,
 390         0xa008, 0xffffffff, 0x00010000,
 391         0xa008, 0xffffffff, 0x00010000,
 392         0x913c, 0xffff03ff, 0x01000100,
 393         0x913c, 0xffff03ff, 0x01000100,
 394         0x90e8, 0x001fffff, 0x010400c0,
 395         0x8c00, 0x000000ff, 0x00000003,
 396         0x8c00, 0x000000ff, 0x00000003,
 397         0x8c04, 0xf8ff00ff, 0x40600060,
 398         0x8c04, 0xf8ff00ff, 0x40600060,
 399         0x8c30, 0x0000000f, 0x00040005,
 400         0x8cf0, 0x1fff1fff, 0x08e00410,
 401         0x8cf0, 0x1fff1fff, 0x08e00410,
 402         0x900c, 0x00ffffff, 0x0017071f,
 403         0x28350, 0x00000f01, 0x00000000,
 404         0x28350, 0x00000f01, 0x00000000,
 405         0x9508, 0xf700071f, 0x00000002,
 406         0x9508, 0xf700071f, 0x00000002,
 407         0x9688, 0x00300000, 0x0017000f,
 408         0x960c, 0xffffffff, 0x54763210,
 409         0x960c, 0xffffffff, 0x54763210,
 410         0x20ef8, 0x01ff01ff, 0x00000002,
 411         0x20e98, 0xfffffbff, 0x00200000,
 412         0x2015c, 0xffffffff, 0x00000f40,
 413         0x88c4, 0x001f3ae3, 0x00000082,
 414         0x88c4, 0x001f3ae3, 0x00000082,
 415         0x8978, 0x3fffffff, 0x04050140,
 416         0x8978, 0x3fffffff, 0x04050140,
 417         0x88d4, 0x0000001f, 0x00000010,
 418         0x88d4, 0x0000001f, 0x00000010,
 419         0x8974, 0xffffffff, 0x00000000,
 420         0x8974, 0xffffffff, 0x00000000
 421 };
 422
 423 static void ni_init_golden_registers(struct radeon_device *rdev)
 424 {
 425         switch (rdev->family) {
 426         case CHIP_CAYMAN:
 427                 radeon_program_register_sequence(rdev,
 428                                                  cayman_golden_registers,
 429                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 430                 radeon_program_register_sequence(rdev,
 431                                                  cayman_golden_registers2,
 432                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 433                 break;
 434         case CHIP_ARUBA:
 435                 if ((rdev->pdev->device == 0x9900) ||
 436                     (rdev->pdev->device == 0x9901) ||
 437                     (rdev->pdev->device == 0x9903) ||
 438                     (rdev->pdev->device == 0x9904) ||
 439                     (rdev->pdev->device == 0x9905) ||
 440                     (rdev->pdev->device == 0x9906) ||
 441                     (rdev->pdev->device == 0x9907) ||
 442                     (rdev->pdev->device == 0x9908) ||
 443                     (rdev->pdev->device == 0x9909) ||
 444                     (rdev->pdev->device == 0x990A) ||
 445                     (rdev->pdev->device == 0x990B) ||
 446                     (rdev->pdev->device == 0x990C) ||
 447                     (rdev->pdev->device == 0x990D) ||
 448                     (rdev->pdev->device == 0x990E) ||
 449                     (rdev->pdev->device == 0x990F) ||
 450                     (rdev->pdev->device == 0x9910) ||
 451                     (rdev->pdev->device == 0x9913) ||
 452                     (rdev->pdev->device == 0x9917) ||
 453                     (rdev->pdev->device == 0x9918)) {
 454                         radeon_program_register_sequence(rdev,
 455                                                          dvst_golden_registers,
 456                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 457                         radeon_program_register_sequence(rdev,
 458                                                          dvst_golden_registers2,
 459                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 460                 } else {
 461                         radeon_program_register_sequence(rdev,
 462                                                          scrapper_golden_registers,
 463                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 464                         radeon_program_register_sequence(rdev,
 465                                                          dvst_golden_registers2,
 466                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 467                 }
 468                 break;
 469         default:
 470                 break;
 471         }
 472 }
 473
 474 #define BTC_IO_MC_REGS_SIZE 29
 475
 476 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 477         {0x00000077, 0xff010100},
 478         {0x00000078, 0x00000000},
 479         {0x00000079, 0x00001434},
 480         {0x0000007a, 0xcc08ec08},
 481         {0x0000007b, 0x00040000},
 482         {0x0000007c, 0x000080c0},
 483         {0x0000007d, 0x09000000},
 484         {0x0000007e, 0x00210404},
 485         {0x00000081, 0x08a8e800},
 486         {0x00000082, 0x00030444},
 487         {0x00000083, 0x00000000},
 488         {0x00000085, 0x00000001},
 489         {0x00000086, 0x00000002},
 490         {0x00000087, 0x48490000},
 491         {0x00000088, 0x20244647},
 492         {0x00000089, 0x00000005},
 493         {0x0000008b, 0x66030000},
 494         {0x0000008c, 0x00006603},
 495         {0x0000008d, 0x00000100},
 496         {0x0000008f, 0x00001c0a},
 497         {0x00000090, 0xff000001},
 498         {0x00000094, 0x00101101},
 499         {0x00000095, 0x00000fff},
 500         {0x00000096, 0x00116fff},
 501         {0x00000097, 0x60010000},
 502         {0x00000098, 0x10010000},
 503         {0x00000099, 0x00006000},
 504         {0x0000009a, 0x00001000},
 505         {0x0000009f, 0x00946a00}
 506 };
 507
 508 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 509         {0x00000077, 0xff010100},
 510         {0x00000078, 0x00000000},
 511         {0x00000079, 0x00001434},
 512         {0x0000007a, 0xcc08ec08},
 513         {0x0000007b, 0x00040000},
 514         {0x0000007c, 0x000080c0},
 515         {0x0000007d, 0x09000000},
 516         {0x0000007e, 0x00210404},
 517         {0x00000081, 0x08a8e800},
 518         {0x00000082, 0x00030444},
 519         {0x00000083, 0x00000000},
 520         {0x00000085, 0x00000001},
 521         {0x00000086, 0x00000002},
 522         {0x00000087, 0x48490000},
 523         {0x00000088, 0x20244647},
 524         {0x00000089, 0x00000005},
 525         {0x0000008b, 0x66030000},
 526         {0x0000008c, 0x00006603},
 527         {0x0000008d, 0x00000100},
 528         {0x0000008f, 0x00001c0a},
 529         {0x00000090, 0xff000001},
 530         {0x00000094, 0x00101101},
 531         {0x00000095, 0x00000fff},
 532         {0x00000096, 0x00116fff},
 533         {0x00000097, 0x60010000},
 534         {0x00000098, 0x10010000},
 535         {0x00000099, 0x00006000},
 536         {0x0000009a, 0x00001000},
 537         {0x0000009f, 0x00936a00}
 538 };
 539
 540 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 541         {0x00000077, 0xff010100},
 542         {0x00000078, 0x00000000},
 543         {0x00000079, 0x00001434},
 544         {0x0000007a, 0xcc08ec08},
 545         {0x0000007b, 0x00040000},
 546         {0x0000007c, 0x000080c0},
 547         {0x0000007d, 0x09000000},
 548         {0x0000007e, 0x00210404},
 549         {0x00000081, 0x08a8e800},
 550         {0x00000082, 0x00030444},
 551         {0x00000083, 0x00000000},
 552         {0x00000085, 0x00000001},
 553         {0x00000086, 0x00000002},
 554         {0x00000087, 0x48490000},
 555         {0x00000088, 0x20244647},
 556         {0x00000089, 0x00000005},
 557         {0x0000008b, 0x66030000},
 558         {0x0000008c, 0x00006603},
 559         {0x0000008d, 0x00000100},
 560         {0x0000008f, 0x00001c0a},
 561         {0x00000090, 0xff000001},
 562         {0x00000094, 0x00101101},
 563         {0x00000095, 0x00000fff},
 564         {0x00000096, 0x00116fff},
 565         {0x00000097, 0x60010000},
 566         {0x00000098, 0x10010000},
 567         {0x00000099, 0x00006000},
 568         {0x0000009a, 0x00001000},
 569         {0x0000009f, 0x00916a00}
 570 };
 571
 572 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 573         {0x00000077, 0xff010100},
 574         {0x00000078, 0x00000000},
 575         {0x00000079, 0x00001434},
 576         {0x0000007a, 0xcc08ec08},
 577         {0x0000007b, 0x00040000},
 578         {0x0000007c, 0x000080c0},
 579         {0x0000007d, 0x09000000},
 580         {0x0000007e, 0x00210404},
 581         {0x00000081, 0x08a8e800},
 582         {0x00000082, 0x00030444},
 583         {0x00000083, 0x00000000},
 584         {0x00000085, 0x00000001},
 585         {0x00000086, 0x00000002},
 586         {0x00000087, 0x48490000},
 587         {0x00000088, 0x20244647},
 588         {0x00000089, 0x00000005},
 589         {0x0000008b, 0x66030000},
 590         {0x0000008c, 0x00006603},
 591         {0x0000008d, 0x00000100},
 592         {0x0000008f, 0x00001c0a},
 593         {0x00000090, 0xff000001},
 594         {0x00000094, 0x00101101},
 595         {0x00000095, 0x00000fff},
 596         {0x00000096, 0x00116fff},
 597         {0x00000097, 0x60010000},
 598         {0x00000098, 0x10010000},
 599         {0x00000099, 0x00006000},
 600         {0x0000009a, 0x00001000},
 601         {0x0000009f, 0x00976b00}
 602 };
 603
 604 int ni_mc_load_microcode(struct radeon_device *rdev)
 605 {
 606         const __be32 *fw_data;
 607         u32 mem_type, running, blackout = 0;
 608         u32 *io_mc_regs;
 609         int i, ucode_size, regs_size;
 610
 611         if (!rdev->mc_fw)
 612                 return -EINVAL;
 613
 614         switch (rdev->family) {
 615         case CHIP_BARTS:
 616                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 617                 ucode_size = BTC_MC_UCODE_SIZE;
 618                 regs_size = BTC_IO_MC_REGS_SIZE;
 619                 break;
 620         case CHIP_TURKS:
 621                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 622                 ucode_size = BTC_MC_UCODE_SIZE;
 623                 regs_size = BTC_IO_MC_REGS_SIZE;
 624                 break;
 625         case CHIP_CAICOS:
 626         default:
 627                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 628                 ucode_size = BTC_MC_UCODE_SIZE;
 629                 regs_size = BTC_IO_MC_REGS_SIZE;
 630                 break;
 631         case CHIP_CAYMAN:
 632                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 633                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 634                 regs_size = BTC_IO_MC_REGS_SIZE;
 635                 break;
 636         }
 637
 638         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 639         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 640
 641         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 642                 if (running) {
 643                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 644                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 645                 }
 646
 647                 /* reset the engine and set to writable */
 648                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 649                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 650
 651                 /* load mc io regs */
 652                 for (i = 0; i < regs_size; i++) {
 653                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 654                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 655                 }
 656                 /* load the MC ucode */
 657                 fw_data = (const __be32 *)rdev->mc_fw->data;
 658                 for (i = 0; i < ucode_size; i++)
 659                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 660
 661                 /* put the engine back into the active state */
 662                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 663                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 664                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 665
 666                 /* wait for training to complete */
 667                 for (i = 0; i < rdev->usec_timeout; i++) {
 668                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 669                                 break;
 670                         udelay(1);
 671                 }
 672
 673                 if (running)
 674                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 675         }
 676
 677         return 0;
 678 }
 679
 680 int ni_init_microcode(struct radeon_device *rdev)
 681 {
 682         const char *chip_name;
 683         const char *rlc_chip_name;
 684         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 685         size_t smc_req_size = 0;
 686         char fw_name[30];
 687         int err;
 688
 689         DRM_DEBUG("\n");
 690
 691         switch (rdev->family) {
 692         case CHIP_BARTS:
 693                 chip_name = "BARTS";
 694                 rlc_chip_name = "BTC";
 695                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 696                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 697                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 698                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 699                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 700                 break;
 701         case CHIP_TURKS:
 702                 chip_name = "TURKS";
 703                 rlc_chip_name = "BTC";
 704                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 705                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 706                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 707                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 708                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 709                 break;
 710         case CHIP_CAICOS:
 711                 chip_name = "CAICOS";
 712                 rlc_chip_name = "BTC";
 713                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 714                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 715                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 716                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 717                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 718                 break;
 719         case CHIP_CAYMAN:
 720                 chip_name = "CAYMAN";
 721                 rlc_chip_name = "CAYMAN";
 722                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 723                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 724                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 725                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 726                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 727                 break;
 728         case CHIP_ARUBA:
 729                 chip_name = "ARUBA";
 730                 rlc_chip_name = "ARUBA";
 731                 /* pfp/me same size as CAYMAN */
 732                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 733                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 734                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 735                 mc_req_size = 0;
 736                 break;
 737         default: BUG();
 738         }
 739
 740         DRM_INFO("Loading %s Microcode\n", chip_name);
 741
 742         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 743         err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 744         if (err)
 745                 goto out;
 746         if (rdev->pfp_fw->size != pfp_req_size) {
 747                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 748                        rdev->pfp_fw->size, fw_name);
 749                 err = -EINVAL;
 750                 goto out;
 751         }
 752
 753         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 754         err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
 755         if (err)
 756                 goto out;
 757         if (rdev->me_fw->size != me_req_size) {
 758                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 759                        rdev->me_fw->size, fw_name);
 760                 err = -EINVAL;
 761         }
 762
 763         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", rlc_chip_name);
 764         err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 765         if (err)
 766                 goto out;
 767         if (rdev->rlc_fw->size != rlc_req_size) {
 768                 pr_err("ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 769                        rdev->rlc_fw->size, fw_name);
 770                 err = -EINVAL;
 771         }
 772
 773         /* no MC ucode on TN */
 774         if (!(rdev->flags & RADEON_IS_IGP)) {
 775                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 776                 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 777                 if (err)
 778                         goto out;
 779                 if (rdev->mc_fw->size != mc_req_size) {
 780                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 781                                rdev->mc_fw->size, fw_name);
 782                         err = -EINVAL;
 783                 }
 784         }
 785
 786         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 787                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 788                 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 789                 if (err) {
 790                         pr_err("smc: error loading firmware \"%s\"\n", fw_name);
 791                         release_firmware(rdev->smc_fw);
 792                         rdev->smc_fw = NULL;
 793                         err = 0;
 794                 } else if (rdev->smc_fw->size != smc_req_size) {
 795                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 796                                rdev->mc_fw->size, fw_name);
 797                         err = -EINVAL;
 798                 }
 799         }
 800
 801 out:
 802         if (err) {
 803                 if (err != -EINVAL)
 804                         pr_err("ni_cp: Failed to load firmware \"%s\"\n",
 805                                fw_name);
 806                 release_firmware(rdev->pfp_fw);
 807                 rdev->pfp_fw = NULL;
 808                 release_firmware(rdev->me_fw);
 809                 rdev->me_fw = NULL;
 810                 release_firmware(rdev->rlc_fw);
 811                 rdev->rlc_fw = NULL;
 812                 release_firmware(rdev->mc_fw);
 813                 rdev->mc_fw = NULL;
 814         }
 815         return err;
 816 }
 817
 818 /**
 819  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 820  *
 821  * @rdev: radeon_device pointer
 822  * @reg: register offset in bytes
 823  * @val: register value
 824  *
 825  * Returns 0 for success or -EINVAL for an invalid register
 826  *
 827  */
 828 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 829                                      u32 reg, u32 *val)
 830 {
 831         switch (reg) {
 832         case GRBM_STATUS:
 833         case GRBM_STATUS_SE0:
 834         case GRBM_STATUS_SE1:
 835         case SRBM_STATUS:
 836         case SRBM_STATUS2:
 837         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 838         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 839         case UVD_STATUS:
 840                 *val = RREG32(reg);
 841                 return 0;
 842         default:
 843                 return -EINVAL;
 844         }
 845 }
 846
 847 int tn_get_temp(struct radeon_device *rdev)
 848 {
 849         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 850         int actual_temp = (temp / 8) - 49;
 851
 852         return actual_temp * 1000;
 853 }
 854
 855 /*
 856  * Core functions
 857  */
 858 static void cayman_gpu_init(struct radeon_device *rdev)
 859 {
 860         u32 gb_addr_config = 0;
 861         u32 mc_arb_ramcfg;
 862         u32 cgts_tcc_disable;
 863         u32 sx_debug_1;
 864         u32 smx_dc_ctl0;
 865         u32 cgts_sm_ctrl_reg;
 866         u32 hdp_host_path_cntl;
 867         u32 tmp;
 868         u32 disabled_rb_mask;
 869         int i, j;
 870
 871         switch (rdev->family) {
 872         case CHIP_CAYMAN:
 873                 rdev->config.cayman.max_shader_engines = 2;
 874                 rdev->config.cayman.max_pipes_per_simd = 4;
 875                 rdev->config.cayman.max_tile_pipes = 8;
 876                 rdev->config.cayman.max_simds_per_se = 12;
 877                 rdev->config.cayman.max_backends_per_se = 4;
 878                 rdev->config.cayman.max_texture_channel_caches = 8;
 879                 rdev->config.cayman.max_gprs = 256;
 880                 rdev->config.cayman.max_threads = 256;
 881                 rdev->config.cayman.max_gs_threads = 32;
 882                 rdev->config.cayman.max_stack_entries = 512;
 883                 rdev->config.cayman.sx_num_of_sets = 8;
 884                 rdev->config.cayman.sx_max_export_size = 256;
 885                 rdev->config.cayman.sx_max_export_pos_size = 64;
 886                 rdev->config.cayman.sx_max_export_smx_size = 192;
 887                 rdev->config.cayman.max_hw_contexts = 8;
 888                 rdev->config.cayman.sq_num_cf_insts = 2;
 889
 890                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 891                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 892                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 893                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 894                 break;
 895         case CHIP_ARUBA:
 896         default:
 897                 rdev->config.cayman.max_shader_engines = 1;
 898                 rdev->config.cayman.max_pipes_per_simd = 4;
 899                 rdev->config.cayman.max_tile_pipes = 2;
 900                 if ((rdev->pdev->device == 0x9900) ||
 901                     (rdev->pdev->device == 0x9901) ||
 902                     (rdev->pdev->device == 0x9905) ||
 903                     (rdev->pdev->device == 0x9906) ||
 904                     (rdev->pdev->device == 0x9907) ||
 905                     (rdev->pdev->device == 0x9908) ||
 906                     (rdev->pdev->device == 0x9909) ||
 907                     (rdev->pdev->device == 0x990B) ||
 908                     (rdev->pdev->device == 0x990C) ||
 909                     (rdev->pdev->device == 0x990F) ||
 910                     (rdev->pdev->device == 0x9910) ||
 911                     (rdev->pdev->device == 0x9917) ||
 912                     (rdev->pdev->device == 0x9999) ||
 913                     (rdev->pdev->device == 0x999C)) {
 914                         rdev->config.cayman.max_simds_per_se = 6;
 915                         rdev->config.cayman.max_backends_per_se = 2;
 916                         rdev->config.cayman.max_hw_contexts = 8;
 917                         rdev->config.cayman.sx_max_export_size = 256;
 918                         rdev->config.cayman.sx_max_export_pos_size = 64;
 919                         rdev->config.cayman.sx_max_export_smx_size = 192;
 920                 } else if ((rdev->pdev->device == 0x9903) ||
 921                            (rdev->pdev->device == 0x9904) ||
 922                            (rdev->pdev->device == 0x990A) ||
 923                            (rdev->pdev->device == 0x990D) ||
 924                            (rdev->pdev->device == 0x990E) ||
 925                            (rdev->pdev->device == 0x9913) ||
 926                            (rdev->pdev->device == 0x9918) ||
 927                            (rdev->pdev->device == 0x999D)) {
 928                         rdev->config.cayman.max_simds_per_se = 4;
 929                         rdev->config.cayman.max_backends_per_se = 2;
 930                         rdev->config.cayman.max_hw_contexts = 8;
 931                         rdev->config.cayman.sx_max_export_size = 256;
 932                         rdev->config.cayman.sx_max_export_pos_size = 64;
 933                         rdev->config.cayman.sx_max_export_smx_size = 192;
 934                 } else if ((rdev->pdev->device == 0x9919) ||
 935                            (rdev->pdev->device == 0x9990) ||
 936                            (rdev->pdev->device == 0x9991) ||
 937                            (rdev->pdev->device == 0x9994) ||
 938                            (rdev->pdev->device == 0x9995) ||
 939                            (rdev->pdev->device == 0x9996) ||
 940                            (rdev->pdev->device == 0x999A) ||
 941                            (rdev->pdev->device == 0x99A0)) {
 942                         rdev->config.cayman.max_simds_per_se = 3;
 943                         rdev->config.cayman.max_backends_per_se = 1;
 944                         rdev->config.cayman.max_hw_contexts = 4;
 945                         rdev->config.cayman.sx_max_export_size = 128;
 946                         rdev->config.cayman.sx_max_export_pos_size = 32;
 947                         rdev->config.cayman.sx_max_export_smx_size = 96;
 948                 } else {
 949                         rdev->config.cayman.max_simds_per_se = 2;
 950                         rdev->config.cayman.max_backends_per_se = 1;
 951                         rdev->config.cayman.max_hw_contexts = 4;
 952                         rdev->config.cayman.sx_max_export_size = 128;
 953                         rdev->config.cayman.sx_max_export_pos_size = 32;
 954                         rdev->config.cayman.sx_max_export_smx_size = 96;
 955                 }
 956                 rdev->config.cayman.max_texture_channel_caches = 2;
 957                 rdev->config.cayman.max_gprs = 256;
 958                 rdev->config.cayman.max_threads = 256;
 959                 rdev->config.cayman.max_gs_threads = 32;
 960                 rdev->config.cayman.max_stack_entries = 512;
 961                 rdev->config.cayman.sx_num_of_sets = 8;
 962                 rdev->config.cayman.sq_num_cf_insts = 2;
 963
 964                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 965                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 966                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 967                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 968                 break;
 969         }
 970
 971         /* Initialize HDP */
 972         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 973                 WREG32((0x2c14 + j), 0x00000000);
 974                 WREG32((0x2c18 + j), 0x00000000);
 975                 WREG32((0x2c1c + j), 0x00000000);
 976                 WREG32((0x2c20 + j), 0x00000000);
 977                 WREG32((0x2c24 + j), 0x00000000);
 978         }
 979
 980         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 981         WREG32(SRBM_INT_CNTL, 0x1);
 982         WREG32(SRBM_INT_ACK, 0x1);
 983
 984         evergreen_fix_pci_max_read_req_size(rdev);
 985
 986         RREG32(MC_SHARED_CHMAP);
 987         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 988
 989         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 990         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 991         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 992                 rdev->config.cayman.mem_row_size_in_kb = 4;
 993         /* XXX use MC settings? */
 994         rdev->config.cayman.shader_engine_tile_size = 32;
 995         rdev->config.cayman.num_gpus = 1;
 996         rdev->config.cayman.multi_gpu_tile_size = 64;
 997
 998         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 999         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1000         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1001         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1002         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1003         rdev->config.cayman.num_shader_engines = tmp + 1;
1004         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1005         rdev->config.cayman.num_gpus = tmp + 1;
1006         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1007         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1008         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1009         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1010
1011
1012         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1013          * not have bank info, so create a custom tiling dword.
1014          * bits 3:0   num_pipes
1015          * bits 7:4   num_banks
1016          * bits 11:8  group_size
1017          * bits 15:12 row_size
1018          */
1019         rdev->config.cayman.tile_config = 0;
1020         switch (rdev->config.cayman.num_tile_pipes) {
1021         case 1:
1022         default:
1023                 rdev->config.cayman.tile_config |= (0 << 0);
1024                 break;
1025         case 2:
1026                 rdev->config.cayman.tile_config |= (1 << 0);
1027                 break;
1028         case 4:
1029                 rdev->config.cayman.tile_config |= (2 << 0);
1030                 break;
1031         case 8:
1032                 rdev->config.cayman.tile_config |= (3 << 0);
1033                 break;
1034         }
1035
1036         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1037         if (rdev->flags & RADEON_IS_IGP)
1038                 rdev->config.cayman.tile_config |= 1 << 4;
1039         else {
1040                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1041                 case 0: /* four banks */
1042                         rdev->config.cayman.tile_config |= 0 << 4;
1043                         break;
1044                 case 1: /* eight banks */
1045                         rdev->config.cayman.tile_config |= 1 << 4;
1046                         break;
1047                 case 2: /* sixteen banks */
1048                 default:
1049                         rdev->config.cayman.tile_config |= 2 << 4;
1050                         break;
1051                 }
1052         }
1053         rdev->config.cayman.tile_config |=
1054                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1055         rdev->config.cayman.tile_config |=
1056                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1057
1058         tmp = 0;
1059         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1060                 u32 rb_disable_bitmap;
1061
1062                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1063                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1064                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1065                 tmp <<= 4;
1066                 tmp |= rb_disable_bitmap;
1067         }
1068         /* enabled rb are just the one not disabled :) */
1069         disabled_rb_mask = tmp;
1070         tmp = 0;
1071         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1072                 tmp |= (1 << i);
1073         /* if all the backends are disabled, fix it up here */
1074         if ((disabled_rb_mask & tmp) == tmp) {
1075                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1076                         disabled_rb_mask &= ~(1 << i);
1077         }
1078
1079         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1080                 u32 simd_disable_bitmap;
1081
1082                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1083                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1084                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1085                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1086                 tmp <<= 16;
1087                 tmp |= simd_disable_bitmap;
1088         }
1089         rdev->config.cayman.active_simds = hweight32(~tmp);
1090
1091         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1092         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1093
1094         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1095         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1096         if (ASIC_IS_DCE6(rdev))
1097                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1098         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1099         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1100         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1101         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1102         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1103         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1104
1105         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1106             (rdev->flags & RADEON_IS_IGP)) {
1107                 if ((disabled_rb_mask & 3) == 2) {
1108                         /* RB1 disabled, RB0 enabled */
1109                         tmp = 0x00000000;
1110                 } else {
1111                         /* RB0 disabled, RB1 enabled */
1112                         tmp = 0x11111111;
1113                 }
1114         } else {
1115                 tmp = gb_addr_config & NUM_PIPES_MASK;
1116                 tmp = r6xx_remap_render_backend(rdev, tmp,
1117                                                 rdev->config.cayman.max_backends_per_se *
1118                                                 rdev->config.cayman.max_shader_engines,
1119                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1120         }
1121         rdev->config.cayman.backend_map = tmp;
1122         WREG32(GB_BACKEND_MAP, tmp);
1123
1124         cgts_tcc_disable = 0xffff0000;
1125         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1126                 cgts_tcc_disable &= ~(1 << (16 + i));
1127         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1128         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1129         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1130         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1131
1132         /* reprogram the shader complex */
1133         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1134         for (i = 0; i < 16; i++)
1135                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1136         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1137
1138         /* set HW defaults for 3D engine */
1139         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1140
1141         sx_debug_1 = RREG32(SX_DEBUG_1);
1142         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1143         WREG32(SX_DEBUG_1, sx_debug_1);
1144
1145         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1146         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1147         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1148         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1149
1150         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1151
1152         /* need to be explicitly zero-ed */
1153         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1154         WREG32(SQ_LSTMP_RING_BASE, 0);
1155         WREG32(SQ_HSTMP_RING_BASE, 0);
1156         WREG32(SQ_ESTMP_RING_BASE, 0);
1157         WREG32(SQ_GSTMP_RING_BASE, 0);
1158         WREG32(SQ_VSTMP_RING_BASE, 0);
1159         WREG32(SQ_PSTMP_RING_BASE, 0);
1160
1161         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1162
1163         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1164                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1165                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1166
1167         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1168                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1169                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1170
1171
1172         WREG32(VGT_NUM_INSTANCES, 1);
1173
1174         WREG32(CP_PERFMON_CNTL, 0);
1175
1176         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1177                                   FETCH_FIFO_HIWATER(0x4) |
1178                                   DONE_FIFO_HIWATER(0xe0) |
1179                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1180
1181         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1182         WREG32(SQ_CONFIG, (VC_ENABLE |
1183                            EXPORT_SRC_C |
1184                            GFX_PRIO(0) |
1185                            CS1_PRIO(0) |
1186                            CS2_PRIO(1)));
1187         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1188
1189         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1190                                           FORCE_EOV_MAX_REZ_CNT(255)));
1191
1192         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1193                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1194
1195         WREG32(VGT_GS_VERTEX_REUSE, 16);
1196         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1197
1198         WREG32(CB_PERF_CTR0_SEL_0, 0);
1199         WREG32(CB_PERF_CTR0_SEL_1, 0);
1200         WREG32(CB_PERF_CTR1_SEL_0, 0);
1201         WREG32(CB_PERF_CTR1_SEL_1, 0);
1202         WREG32(CB_PERF_CTR2_SEL_0, 0);
1203         WREG32(CB_PERF_CTR2_SEL_1, 0);
1204         WREG32(CB_PERF_CTR3_SEL_0, 0);
1205         WREG32(CB_PERF_CTR3_SEL_1, 0);
1206
1207         tmp = RREG32(HDP_MISC_CNTL);
1208         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1209         WREG32(HDP_MISC_CNTL, tmp);
1210
1211         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1212         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1213
1214         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1215
1216         udelay(50);
1217
1218         /* set clockgating golden values on TN */
1219         if (rdev->family == CHIP_ARUBA) {
1220                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1221                 tmp &= ~0x00380000;
1222                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1223                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1224                 tmp &= ~0x0e000000;
1225                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1226         }
1227 }
1228
1229 /*
1230  * GART
1231  */
1232 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1233 {
1234         /* flush hdp cache */
1235         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1236
1237         /* bits 0-7 are the VM contexts0-7 */
1238         WREG32(VM_INVALIDATE_REQUEST, 1);
1239 }
1240
1241 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1242 {
1243         int i, r;
1244
1245         if (rdev->gart.robj == NULL) {
1246                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1247                 return -EINVAL;
1248         }
1249         r = radeon_gart_table_vram_pin(rdev);
1250         if (r)
1251                 return r;
1252         /* Setup TLB control */
1253         WREG32(MC_VM_MX_L1_TLB_CNTL,
1254                (0xA << 7) |
1255                ENABLE_L1_TLB |
1256                ENABLE_L1_FRAGMENT_PROCESSING |
1257                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1258                ENABLE_ADVANCED_DRIVER_MODEL |
1259                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1260         /* Setup L2 cache */
1261         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1262                ENABLE_L2_FRAGMENT_PROCESSING |
1263                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1264                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1265                EFFECTIVE_L2_QUEUE_SIZE(7) |
1266                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1267         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1268         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1269                BANK_SELECT(6) |
1270                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1271         /* setup context0 */
1272         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1273         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1274         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1275         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1276                         (u32)(rdev->dummy_page.addr >> 12));
1277         WREG32(VM_CONTEXT0_CNTL2, 0);
1278         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1279                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1280
1281         WREG32(0x15D4, 0);
1282         WREG32(0x15D8, 0);
1283         WREG32(0x15DC, 0);
1284
1285         /* empty context1-7 */
1286         /* Assign the pt base to something valid for now; the pts used for
1287          * the VMs are determined by the application and setup and assigned
1288          * on the fly in the vm part of radeon_gart.c
1289          */
1290         for (i = 1; i < 8; i++) {
1291                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1292                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1293                         rdev->vm_manager.max_pfn - 1);
1294                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1295                        rdev->vm_manager.saved_table_addr[i]);
1296         }
1297
1298         /* enable context1-7 */
1299         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1300                (u32)(rdev->dummy_page.addr >> 12));
1301         WREG32(VM_CONTEXT1_CNTL2, 4);
1302         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1303                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1304                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1305                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1306                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1307                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1308                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1309                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1310                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1311                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1312                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1313                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1314                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1315                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1316
1317         cayman_pcie_gart_tlb_flush(rdev);
1318         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1319                  (unsigned)(rdev->mc.gtt_size >> 20),
1320                  (unsigned long long)rdev->gart.table_addr);
1321         rdev->gart.ready = true;
1322         return 0;
1323 }
1324
1325 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1326 {
1327         unsigned i;
1328
1329         for (i = 1; i < 8; ++i) {
1330                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1331                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1332         }
1333
1334         /* Disable all tables */
1335         WREG32(VM_CONTEXT0_CNTL, 0);
1336         WREG32(VM_CONTEXT1_CNTL, 0);
1337         /* Setup TLB control */
1338         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1339                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1340                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1341         /* Setup L2 cache */
1342         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1343                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1344                EFFECTIVE_L2_QUEUE_SIZE(7) |
1345                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1346         WREG32(VM_L2_CNTL2, 0);
1347         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1348                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1349         radeon_gart_table_vram_unpin(rdev);
1350 }
1351
1352 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1353 {
1354         cayman_pcie_gart_disable(rdev);
1355         radeon_gart_table_vram_free(rdev);
1356         radeon_gart_fini(rdev);
1357 }
1358
1359 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1360                               int ring, u32 cp_int_cntl)
1361 {
1362         WREG32(SRBM_GFX_CNTL, RINGID(ring));
1363         WREG32(CP_INT_CNTL, cp_int_cntl);
1364 }
1365
1366 /*
1367  * CP.
1368  */
1369 void cayman_fence_ring_emit(struct radeon_device *rdev,
1370                             struct radeon_fence *fence)
1371 {
1372         struct radeon_ring *ring = &rdev->ring[fence->ring];
1373         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1374         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1375                 PACKET3_SH_ACTION_ENA;
1376
1377         /* flush read cache over gart for this vmid */
1378         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1379         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1380         radeon_ring_write(ring, 0xFFFFFFFF);
1381         radeon_ring_write(ring, 0);
1382         radeon_ring_write(ring, 10); /* poll interval */
1383         /* EVENT_WRITE_EOP - flush caches, send int */
1384         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1385         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1386         radeon_ring_write(ring, lower_32_bits(addr));
1387         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1388         radeon_ring_write(ring, fence->seq);
1389         radeon_ring_write(ring, 0);
1390 }
1391
1392 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1393 {
1394         struct radeon_ring *ring = &rdev->ring[ib->ring];
1395         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1396         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1397                 PACKET3_SH_ACTION_ENA;
1398
1399         /* set to DX10/11 mode */
1400         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1401         radeon_ring_write(ring, 1);
1402
1403         if (ring->rptr_save_reg) {
1404                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1405                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1406                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1407                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1408                 radeon_ring_write(ring, next_rptr);
1409         }
1410
1411         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1412         radeon_ring_write(ring,
1413 #ifdef __BIG_ENDIAN
1414                           (2 << 0) |
1415 #endif
1416                           (ib->gpu_addr & 0xFFFFFFFC));
1417         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1418         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1419
1420         /* flush read cache over gart for this vmid */
1421         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1422         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1423         radeon_ring_write(ring, 0xFFFFFFFF);
1424         radeon_ring_write(ring, 0);
1425         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1426 }
1427
1428 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1429 {
1430         if (enable)
1431                 WREG32(CP_ME_CNTL, 0);
1432         else {
1433                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1434                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1435                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1436                 WREG32(SCRATCH_UMSK, 0);
1437                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1438         }
1439 }
1440
1441 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1442                         struct radeon_ring *ring)
1443 {
1444         u32 rptr;
1445
1446         if (rdev->wb.enabled)
1447                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1448         else {
1449                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1450                         rptr = RREG32(CP_RB0_RPTR);
1451                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1452                         rptr = RREG32(CP_RB1_RPTR);
1453                 else
1454                         rptr = RREG32(CP_RB2_RPTR);
1455         }
1456
1457         return rptr;
1458 }
1459
1460 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1461                         struct radeon_ring *ring)
1462 {
1463         u32 wptr;
1464
1465         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1466                 wptr = RREG32(CP_RB0_WPTR);
1467         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1468                 wptr = RREG32(CP_RB1_WPTR);
1469         else
1470                 wptr = RREG32(CP_RB2_WPTR);
1471
1472         return wptr;
1473 }
1474
1475 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1476                          struct radeon_ring *ring)
1477 {
1478         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1479                 WREG32(CP_RB0_WPTR, ring->wptr);
1480                 (void)RREG32(CP_RB0_WPTR);
1481         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1482                 WREG32(CP_RB1_WPTR, ring->wptr);
1483                 (void)RREG32(CP_RB1_WPTR);
1484         } else {
1485                 WREG32(CP_RB2_WPTR, ring->wptr);
1486                 (void)RREG32(CP_RB2_WPTR);
1487         }
1488 }
1489
1490 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1491 {
1492         const __be32 *fw_data;
1493         int i;
1494
1495         if (!rdev->me_fw || !rdev->pfp_fw)
1496                 return -EINVAL;
1497
1498         cayman_cp_enable(rdev, false);
1499
1500         fw_data = (const __be32 *)rdev->pfp_fw->data;
1501         WREG32(CP_PFP_UCODE_ADDR, 0);
1502         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1503                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1504         WREG32(CP_PFP_UCODE_ADDR, 0);
1505
1506         fw_data = (const __be32 *)rdev->me_fw->data;
1507         WREG32(CP_ME_RAM_WADDR, 0);
1508         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1509                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1510
1511         WREG32(CP_PFP_UCODE_ADDR, 0);
1512         WREG32(CP_ME_RAM_WADDR, 0);
1513         WREG32(CP_ME_RAM_RADDR, 0);
1514         return 0;
1515 }
1516
1517 static int cayman_cp_start(struct radeon_device *rdev)
1518 {
1519         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1520         int r, i;
1521
1522         r = radeon_ring_lock(rdev, ring, 7);
1523         if (r) {
1524                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1525                 return r;
1526         }
1527         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1528         radeon_ring_write(ring, 0x1);
1529         radeon_ring_write(ring, 0x0);
1530         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1531         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1532         radeon_ring_write(ring, 0);
1533         radeon_ring_write(ring, 0);
1534         radeon_ring_unlock_commit(rdev, ring, false);
1535
1536         cayman_cp_enable(rdev, true);
1537
1538         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1539         if (r) {
1540                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1541                 return r;
1542         }
1543
1544         /* setup clear context state */
1545         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1546         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1547
1548         for (i = 0; i < cayman_default_size; i++)
1549                 radeon_ring_write(ring, cayman_default_state[i]);
1550
1551         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1552         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1553
1554         /* set clear context state */
1555         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1556         radeon_ring_write(ring, 0);
1557
1558         /* SQ_VTX_BASE_VTX_LOC */
1559         radeon_ring_write(ring, 0xc0026f00);
1560         radeon_ring_write(ring, 0x00000000);
1561         radeon_ring_write(ring, 0x00000000);
1562         radeon_ring_write(ring, 0x00000000);
1563
1564         /* Clear consts */
1565         radeon_ring_write(ring, 0xc0036f00);
1566         radeon_ring_write(ring, 0x00000bc4);
1567         radeon_ring_write(ring, 0xffffffff);
1568         radeon_ring_write(ring, 0xffffffff);
1569         radeon_ring_write(ring, 0xffffffff);
1570
1571         radeon_ring_write(ring, 0xc0026900);
1572         radeon_ring_write(ring, 0x00000316);
1573         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1574         radeon_ring_write(ring, 0x00000010); /*  */
1575
1576         radeon_ring_unlock_commit(rdev, ring, false);
1577
1578         /* XXX init other rings */
1579
1580         return 0;
1581 }
1582
1583 static void cayman_cp_fini(struct radeon_device *rdev)
1584 {
1585         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1586         cayman_cp_enable(rdev, false);
1587         radeon_ring_fini(rdev, ring);
1588         radeon_scratch_free(rdev, ring->rptr_save_reg);
1589 }
1590
1591 static int cayman_cp_resume(struct radeon_device *rdev)
1592 {
1593         static const int ridx[] = {
1594                 RADEON_RING_TYPE_GFX_INDEX,
1595                 CAYMAN_RING_TYPE_CP1_INDEX,
1596                 CAYMAN_RING_TYPE_CP2_INDEX
1597         };
1598         static const unsigned cp_rb_cntl[] = {
1599                 CP_RB0_CNTL,
1600                 CP_RB1_CNTL,
1601                 CP_RB2_CNTL,
1602         };
1603         static const unsigned cp_rb_rptr_addr[] = {
1604                 CP_RB0_RPTR_ADDR,
1605                 CP_RB1_RPTR_ADDR,
1606                 CP_RB2_RPTR_ADDR
1607         };
1608         static const unsigned cp_rb_rptr_addr_hi[] = {
1609                 CP_RB0_RPTR_ADDR_HI,
1610                 CP_RB1_RPTR_ADDR_HI,
1611                 CP_RB2_RPTR_ADDR_HI
1612         };
1613         static const unsigned cp_rb_base[] = {
1614                 CP_RB0_BASE,
1615                 CP_RB1_BASE,
1616                 CP_RB2_BASE
1617         };
1618         static const unsigned cp_rb_rptr[] = {
1619                 CP_RB0_RPTR,
1620                 CP_RB1_RPTR,
1621                 CP_RB2_RPTR
1622         };
1623         static const unsigned cp_rb_wptr[] = {
1624                 CP_RB0_WPTR,
1625                 CP_RB1_WPTR,
1626                 CP_RB2_WPTR
1627         };
1628         struct radeon_ring *ring;
1629         int i, r;
1630
1631         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1632         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1633                                  SOFT_RESET_PA |
1634                                  SOFT_RESET_SH |
1635                                  SOFT_RESET_VGT |
1636                                  SOFT_RESET_SPI |
1637                                  SOFT_RESET_SX));
1638         RREG32(GRBM_SOFT_RESET);
1639         mdelay(15);
1640         WREG32(GRBM_SOFT_RESET, 0);
1641         RREG32(GRBM_SOFT_RESET);
1642
1643         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1644         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1645
1646         /* Set the write pointer delay */
1647         WREG32(CP_RB_WPTR_DELAY, 0);
1648
1649         WREG32(CP_DEBUG, (1 << 27));
1650
1651         /* set the wb address whether it's enabled or not */
1652         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1653         WREG32(SCRATCH_UMSK, 0xff);
1654
1655         for (i = 0; i < 3; ++i) {
1656                 uint32_t rb_cntl;
1657                 uint64_t addr;
1658
1659                 /* Set ring buffer size */
1660                 ring = &rdev->ring[ridx[i]];
1661                 rb_cntl = order_base_2(ring->ring_size / 8);
1662                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1663 #ifdef __BIG_ENDIAN
1664                 rb_cntl |= BUF_SWAP_32BIT;
1665 #endif
1666                 WREG32(cp_rb_cntl[i], rb_cntl);
1667
1668                 /* set the wb address whether it's enabled or not */
1669                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1670                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1671                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1672         }
1673
1674         /* set the rb base addr, this causes an internal reset of ALL rings */
1675         for (i = 0; i < 3; ++i) {
1676                 ring = &rdev->ring[ridx[i]];
1677                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1678         }
1679
1680         for (i = 0; i < 3; ++i) {
1681                 /* Initialize the ring buffer's read and write pointers */
1682                 ring = &rdev->ring[ridx[i]];
1683                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1684
1685                 ring->wptr = 0;
1686                 WREG32(cp_rb_rptr[i], 0);
1687                 WREG32(cp_rb_wptr[i], ring->wptr);
1688
1689                 mdelay(1);
1690                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1691         }
1692
1693         /* start the rings */
1694         cayman_cp_start(rdev);
1695         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1696         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1697         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1698         /* this only test cp0 */
1699         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1700         if (r) {
1701                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1702                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1703                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1704                 return r;
1705         }
1706
1707         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1708                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1709
1710         return 0;
1711 }
1712
1713 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1714 {
1715         u32 reset_mask = 0;
1716         u32 tmp;
1717
1718         /* GRBM_STATUS */
1719         tmp = RREG32(GRBM_STATUS);
1720         if (tmp & (PA_BUSY | SC_BUSY |
1721                    SH_BUSY | SX_BUSY |
1722                    TA_BUSY | VGT_BUSY |
1723                    DB_BUSY | CB_BUSY |
1724                    GDS_BUSY | SPI_BUSY |
1725                    IA_BUSY | IA_BUSY_NO_DMA))
1726                 reset_mask |= RADEON_RESET_GFX;
1727
1728         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1729                    CP_BUSY | CP_COHERENCY_BUSY))
1730                 reset_mask |= RADEON_RESET_CP;
1731
1732         if (tmp & GRBM_EE_BUSY)
1733                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1734
1735         /* DMA_STATUS_REG 0 */
1736         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1737         if (!(tmp & DMA_IDLE))
1738                 reset_mask |= RADEON_RESET_DMA;
1739
1740         /* DMA_STATUS_REG 1 */
1741         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1742         if (!(tmp & DMA_IDLE))
1743                 reset_mask |= RADEON_RESET_DMA1;
1744
1745         /* SRBM_STATUS2 */
1746         tmp = RREG32(SRBM_STATUS2);
1747         if (tmp & DMA_BUSY)
1748                 reset_mask |= RADEON_RESET_DMA;
1749
1750         if (tmp & DMA1_BUSY)
1751                 reset_mask |= RADEON_RESET_DMA1;
1752
1753         /* SRBM_STATUS */
1754         tmp = RREG32(SRBM_STATUS);
1755         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1756                 reset_mask |= RADEON_RESET_RLC;
1757
1758         if (tmp & IH_BUSY)
1759                 reset_mask |= RADEON_RESET_IH;
1760
1761         if (tmp & SEM_BUSY)
1762                 reset_mask |= RADEON_RESET_SEM;
1763
1764         if (tmp & GRBM_RQ_PENDING)
1765                 reset_mask |= RADEON_RESET_GRBM;
1766
1767         if (tmp & VMC_BUSY)
1768                 reset_mask |= RADEON_RESET_VMC;
1769
1770         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1771                    MCC_BUSY | MCD_BUSY))
1772                 reset_mask |= RADEON_RESET_MC;
1773
1774         if (evergreen_is_display_hung(rdev))
1775                 reset_mask |= RADEON_RESET_DISPLAY;
1776
1777         /* VM_L2_STATUS */
1778         tmp = RREG32(VM_L2_STATUS);
1779         if (tmp & L2_BUSY)
1780                 reset_mask |= RADEON_RESET_VMC;
1781
1782         /* Skip MC reset as it's mostly likely not hung, just busy */
1783         if (reset_mask & RADEON_RESET_MC) {
1784                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1785                 reset_mask &= ~RADEON_RESET_MC;
1786         }
1787
1788         return reset_mask;
1789 }
1790
1791 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1792 {
1793         struct evergreen_mc_save save;
1794         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1795         u32 tmp;
1796
1797         if (reset_mask == 0)
1798                 return;
1799
1800         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1801
1802         evergreen_print_gpu_status_regs(rdev);
1803         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1804                  RREG32(0x14F8));
1805         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1806                  RREG32(0x14D8));
1807         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1808                  RREG32(0x14FC));
1809         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1810                  RREG32(0x14DC));
1811
1812         /* Disable CP parsing/prefetching */
1813         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1814
1815         if (reset_mask & RADEON_RESET_DMA) {
1816                 /* dma0 */
1817                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1818                 tmp &= ~DMA_RB_ENABLE;
1819                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1820         }
1821
1822         if (reset_mask & RADEON_RESET_DMA1) {
1823                 /* dma1 */
1824                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1825                 tmp &= ~DMA_RB_ENABLE;
1826                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1827         }
1828
1829         udelay(50);
1830
1831         evergreen_mc_stop(rdev, &save);
1832         if (evergreen_mc_wait_for_idle(rdev)) {
1833                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1834         }
1835
1836         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1837                 grbm_soft_reset = SOFT_RESET_CB |
1838                         SOFT_RESET_DB |
1839                         SOFT_RESET_GDS |
1840                         SOFT_RESET_PA |
1841                         SOFT_RESET_SC |
1842                         SOFT_RESET_SPI |
1843                         SOFT_RESET_SH |
1844                         SOFT_RESET_SX |
1845                         SOFT_RESET_TC |
1846                         SOFT_RESET_TA |
1847                         SOFT_RESET_VGT |
1848                         SOFT_RESET_IA;
1849         }
1850
1851         if (reset_mask & RADEON_RESET_CP) {
1852                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1853
1854                 srbm_soft_reset |= SOFT_RESET_GRBM;
1855         }
1856
1857         if (reset_mask & RADEON_RESET_DMA)
1858                 srbm_soft_reset |= SOFT_RESET_DMA;
1859
1860         if (reset_mask & RADEON_RESET_DMA1)
1861                 srbm_soft_reset |= SOFT_RESET_DMA1;
1862
1863         if (reset_mask & RADEON_RESET_DISPLAY)
1864                 srbm_soft_reset |= SOFT_RESET_DC;
1865
1866         if (reset_mask & RADEON_RESET_RLC)
1867                 srbm_soft_reset |= SOFT_RESET_RLC;
1868
1869         if (reset_mask & RADEON_RESET_SEM)
1870                 srbm_soft_reset |= SOFT_RESET_SEM;
1871
1872         if (reset_mask & RADEON_RESET_IH)
1873                 srbm_soft_reset |= SOFT_RESET_IH;
1874
1875         if (reset_mask & RADEON_RESET_GRBM)
1876                 srbm_soft_reset |= SOFT_RESET_GRBM;
1877
1878         if (reset_mask & RADEON_RESET_VMC)
1879                 srbm_soft_reset |= SOFT_RESET_VMC;
1880
1881         if (!(rdev->flags & RADEON_IS_IGP)) {
1882                 if (reset_mask & RADEON_RESET_MC)
1883                         srbm_soft_reset |= SOFT_RESET_MC;
1884         }
1885
1886         if (grbm_soft_reset) {
1887                 tmp = RREG32(GRBM_SOFT_RESET);
1888                 tmp |= grbm_soft_reset;
1889                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1890                 WREG32(GRBM_SOFT_RESET, tmp);
1891                 tmp = RREG32(GRBM_SOFT_RESET);
1892
1893                 udelay(50);
1894
1895                 tmp &= ~grbm_soft_reset;
1896                 WREG32(GRBM_SOFT_RESET, tmp);
1897                 tmp = RREG32(GRBM_SOFT_RESET);
1898         }
1899
1900         if (srbm_soft_reset) {
1901                 tmp = RREG32(SRBM_SOFT_RESET);
1902                 tmp |= srbm_soft_reset;
1903                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1904                 WREG32(SRBM_SOFT_RESET, tmp);
1905                 tmp = RREG32(SRBM_SOFT_RESET);
1906
1907                 udelay(50);
1908
1909                 tmp &= ~srbm_soft_reset;
1910                 WREG32(SRBM_SOFT_RESET, tmp);
1911                 tmp = RREG32(SRBM_SOFT_RESET);
1912         }
1913
1914         /* Wait a little for things to settle down */
1915         udelay(50);
1916
1917         evergreen_mc_resume(rdev, &save);
1918         udelay(50);
1919
1920         evergreen_print_gpu_status_regs(rdev);
1921 }
1922
1923 int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1924 {
1925         u32 reset_mask;
1926
1927         if (hard) {
1928                 evergreen_gpu_pci_config_reset(rdev);
1929                 return 0;
1930         }
1931
1932         reset_mask = cayman_gpu_check_soft_reset(rdev);
1933
1934         if (reset_mask)
1935                 r600_set_bios_scratch_engine_hung(rdev, true);
1936
1937         cayman_gpu_soft_reset(rdev, reset_mask);
1938
1939         reset_mask = cayman_gpu_check_soft_reset(rdev);
1940
1941         if (reset_mask)
1942                 evergreen_gpu_pci_config_reset(rdev);
1943
1944         r600_set_bios_scratch_engine_hung(rdev, false);
1945
1946         return 0;
1947 }
1948
1949 /**
1950  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1951  *
1952  * @rdev: radeon_device pointer
1953  * @ring: radeon_ring structure holding ring information
1954  *
1955  * Check if the GFX engine is locked up.
1956  * Returns true if the engine appears to be locked up, false if not.
1957  */
1958 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1959 {
1960         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1961
1962         if (!(reset_mask & (RADEON_RESET_GFX |
1963                             RADEON_RESET_COMPUTE |
1964                             RADEON_RESET_CP))) {
1965                 radeon_ring_lockup_update(rdev, ring);
1966                 return false;
1967         }
1968         return radeon_ring_test_lockup(rdev, ring);
1969 }
1970
1971 static void cayman_uvd_init(struct radeon_device *rdev)
1972 {
1973         int r;
1974
1975         if (!rdev->has_uvd)
1976                 return;
1977
1978         r = radeon_uvd_init(rdev);
1979         if (r) {
1980                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1981                 /*
1982                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1983                  * to early fails uvd_v2_2_resume() and thus nothing happens
1984                  * there. So it is pointless to try to go through that code
1985                  * hence why we disable uvd here.
1986                  */
1987                 rdev->has_uvd = false;
1988                 return;
1989         }
1990         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1991         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
1992 }
1993
1994 static void cayman_uvd_start(struct radeon_device *rdev)
1995 {
1996         int r;
1997
1998         if (!rdev->has_uvd)
1999                 return;
2000
2001         r = uvd_v2_2_resume(rdev);
2002         if (r) {
2003                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2004                 goto error;
2005         }
2006         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2007         if (r) {
2008                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2009                 goto error;
2010         }
2011         return;
2012
2013 error:
2014         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2015 }
2016
2017 static void cayman_uvd_resume(struct radeon_device *rdev)
2018 {
2019         struct radeon_ring *ring;
2020         int r;
2021
2022         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2023                 return;
2024
2025         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2026         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2027         if (r) {
2028                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2029                 return;
2030         }
2031         r = uvd_v1_0_init(rdev);
2032         if (r) {
2033                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2034                 return;
2035         }
2036 }
2037
2038 static void cayman_vce_init(struct radeon_device *rdev)
2039 {
2040         int r;
2041
2042         /* Only set for CHIP_ARUBA */
2043         if (!rdev->has_vce)
2044                 return;
2045
2046         r = radeon_vce_init(rdev);
2047         if (r) {
2048                 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2049                 /*
2050                  * At this point rdev->vce.vcpu_bo is NULL which trickles down
2051                  * to early fails cayman_vce_start() and thus nothing happens
2052                  * there. So it is pointless to try to go through that code
2053                  * hence why we disable vce here.
2054                  */
2055                 rdev->has_vce = false;
2056                 return;
2057         }
2058         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2059         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2060         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2061         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2062 }
2063
2064 static void cayman_vce_start(struct radeon_device *rdev)
2065 {
2066         int r;
2067
2068         if (!rdev->has_vce)
2069                 return;
2070
2071         r = radeon_vce_resume(rdev);
2072         if (r) {
2073                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2074                 goto error;
2075         }
2076         r = vce_v1_0_resume(rdev);
2077         if (r) {
2078                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2079                 goto error;
2080         }
2081         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2082         if (r) {
2083                 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2084                 goto error;
2085         }
2086         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2087         if (r) {
2088                 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2089                 goto error;
2090         }
2091         return;
2092
2093 error:
2094         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2095         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2096 }
2097
2098 static void cayman_vce_resume(struct radeon_device *rdev)
2099 {
2100         struct radeon_ring *ring;
2101         int r;
2102
2103         if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2104                 return;
2105
2106         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2107         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2108         if (r) {
2109                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2110                 return;
2111         }
2112         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2113         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2114         if (r) {
2115                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2116                 return;
2117         }
2118         r = vce_v1_0_init(rdev);
2119         if (r) {
2120                 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2121                 return;
2122         }
2123 }
2124
2125 static int cayman_startup(struct radeon_device *rdev)
2126 {
2127         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2128         int r;
2129
2130         /* enable pcie gen2 link */
2131         evergreen_pcie_gen2_enable(rdev);
2132         /* enable aspm */
2133         evergreen_program_aspm(rdev);
2134
2135         /* scratch needs to be initialized before MC */
2136         r = r600_vram_scratch_init(rdev);
2137         if (r)
2138                 return r;
2139
2140         evergreen_mc_program(rdev);
2141
2142         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2143                 r = ni_mc_load_microcode(rdev);
2144                 if (r) {
2145                         DRM_ERROR("Failed to load MC firmware!\n");
2146                         return r;
2147                 }
2148         }
2149
2150         r = cayman_pcie_gart_enable(rdev);
2151         if (r)
2152                 return r;
2153         cayman_gpu_init(rdev);
2154
2155         /* allocate rlc buffers */
2156         if (rdev->flags & RADEON_IS_IGP) {
2157                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2158                 rdev->rlc.reg_list_size =
2159                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2160                 rdev->rlc.cs_data = cayman_cs_data;
2161                 r = sumo_rlc_init(rdev);
2162                 if (r) {
2163                         DRM_ERROR("Failed to init rlc BOs!\n");
2164                         return r;
2165                 }
2166         }
2167
2168         /* allocate wb buffer */
2169         r = radeon_wb_init(rdev);
2170         if (r)
2171                 return r;
2172
2173         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2174         if (r) {
2175                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2176                 return r;
2177         }
2178
2179         cayman_uvd_start(rdev);
2180         cayman_vce_start(rdev);
2181
2182         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2183         if (r) {
2184                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2185                 return r;
2186         }
2187
2188         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2189         if (r) {
2190                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2191                 return r;
2192         }
2193
2194         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2195         if (r) {
2196                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2197                 return r;
2198         }
2199
2200         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2201         if (r) {
2202                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2203                 return r;
2204         }
2205
2206         /* Enable IRQ */
2207         if (!rdev->irq.installed) {
2208                 r = radeon_irq_kms_init(rdev);
2209                 if (r)
2210                         return r;
2211         }
2212
2213         r = r600_irq_init(rdev);
2214         if (r) {
2215                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2216                 radeon_irq_kms_fini(rdev);
2217                 return r;
2218         }
2219         evergreen_irq_set(rdev);
2220
2221         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2222                              RADEON_CP_PACKET2);
2223         if (r)
2224                 return r;
2225
2226         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2227         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2228                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2229         if (r)
2230                 return r;
2231
2232         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2233         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2234                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2235         if (r)
2236                 return r;
2237
2238         r = cayman_cp_load_microcode(rdev);
2239         if (r)
2240                 return r;
2241         r = cayman_cp_resume(rdev);
2242         if (r)
2243                 return r;
2244
2245         r = cayman_dma_resume(rdev);
2246         if (r)
2247                 return r;
2248
2249         cayman_uvd_resume(rdev);
2250         cayman_vce_resume(rdev);
2251
2252         r = radeon_ib_pool_init(rdev);
2253         if (r) {
2254                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2255                 return r;
2256         }
2257
2258         r = radeon_vm_manager_init(rdev);
2259         if (r) {
2260                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2261                 return r;
2262         }
2263
2264         r = radeon_audio_init(rdev);
2265         if (r)
2266                 return r;
2267
2268         return 0;
2269 }
2270
2271 int cayman_resume(struct radeon_device *rdev)
2272 {
2273         int r;
2274
2275         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2276          * posting will perform necessary task to bring back GPU into good
2277          * shape.
2278          */
2279         /* post card */
2280         atom_asic_init(rdev->mode_info.atom_context);
2281
2282         /* init golden registers */
2283         ni_init_golden_registers(rdev);
2284
2285         if (rdev->pm.pm_method == PM_METHOD_DPM)
2286                 radeon_pm_resume(rdev);
2287
2288         rdev->accel_working = true;
2289         r = cayman_startup(rdev);
2290         if (r) {
2291                 DRM_ERROR("cayman startup failed on resume\n");
2292                 rdev->accel_working = false;
2293                 return r;
2294         }
2295         return r;
2296 }
2297
2298 int cayman_suspend(struct radeon_device *rdev)
2299 {
2300         radeon_pm_suspend(rdev);
2301         radeon_audio_fini(rdev);
2302         radeon_vm_manager_fini(rdev);
2303         cayman_cp_enable(rdev, false);
2304         cayman_dma_stop(rdev);
2305         if (rdev->has_uvd) {
2306                 radeon_uvd_suspend(rdev);
2307                 uvd_v1_0_fini(rdev);
2308         }
2309         evergreen_irq_suspend(rdev);
2310         radeon_wb_disable(rdev);
2311         cayman_pcie_gart_disable(rdev);
2312         return 0;
2313 }
2314
2315 /* Plan is to move initialization in that function and use
2316  * helper function so that radeon_device_init pretty much
2317  * do nothing more than calling asic specific function. This
2318  * should also allow to remove a bunch of callback function
2319  * like vram_info.
2320  */
2321 int cayman_init(struct radeon_device *rdev)
2322 {
2323         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2324         int r;
2325
2326         /* Read BIOS */
2327         if (!radeon_get_bios(rdev)) {
2328                 if (ASIC_IS_AVIVO(rdev))
2329                         return -EINVAL;
2330         }
2331         /* Must be an ATOMBIOS */
2332         if (!rdev->is_atom_bios) {
2333                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2334                 return -EINVAL;
2335         }
2336         r = radeon_atombios_init(rdev);
2337         if (r)
2338                 return r;
2339
2340         /* Post card if necessary */
2341         if (!radeon_card_posted(rdev)) {
2342                 if (!rdev->bios) {
2343                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2344                         return -EINVAL;
2345                 }
2346                 DRM_INFO("GPU not posted. posting now...\n");
2347                 atom_asic_init(rdev->mode_info.atom_context);
2348         }
2349         /* init golden registers */
2350         ni_init_golden_registers(rdev);
2351         /* Initialize scratch registers */
2352         r600_scratch_init(rdev);
2353         /* Initialize surface registers */
2354         radeon_surface_init(rdev);
2355         /* Initialize clocks */
2356         radeon_get_clock_info(rdev->ddev);
2357         /* Fence driver */
2358         radeon_fence_driver_init(rdev);
2359         /* initialize memory controller */
2360         r = evergreen_mc_init(rdev);
2361         if (r)
2362                 return r;
2363         /* Memory manager */
2364         r = radeon_bo_init(rdev);
2365         if (r)
2366                 return r;
2367
2368         if (rdev->flags & RADEON_IS_IGP) {
2369                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2370                         r = ni_init_microcode(rdev);
2371                         if (r) {
2372                                 DRM_ERROR("Failed to load firmware!\n");
2373                                 /*(DEBLOBBED)*/
2374                         }
2375                 }
2376         } else {
2377                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2378                         r = ni_init_microcode(rdev);
2379                         if (r) {
2380                                 DRM_ERROR("Failed to load firmware!\n");
2381                                 /*(DEBLOBBED)*/
2382                         }
2383                 }
2384         }
2385
2386         /* Initialize power management */
2387         radeon_pm_init(rdev);
2388
2389         ring->ring_obj = NULL;
2390         r600_ring_init(rdev, ring, 1024 * 1024);
2391
2392         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2393         ring->ring_obj = NULL;
2394         r600_ring_init(rdev, ring, 64 * 1024);
2395
2396         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2397         ring->ring_obj = NULL;
2398         r600_ring_init(rdev, ring, 64 * 1024);
2399
2400         cayman_uvd_init(rdev);
2401         cayman_vce_init(rdev);
2402
2403         rdev->ih.ring_obj = NULL;
2404         r600_ih_ring_init(rdev, 64 * 1024);
2405
2406         r = r600_pcie_gart_init(rdev);
2407         if (r)
2408                 return r;
2409
2410         rdev->accel_working = true;
2411         r = cayman_startup(rdev);
2412         if (r) {
2413                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2414                 cayman_cp_fini(rdev);
2415                 cayman_dma_fini(rdev);
2416                 r600_irq_fini(rdev);
2417                 if (rdev->flags & RADEON_IS_IGP)
2418                         sumo_rlc_fini(rdev);
2419                 radeon_wb_fini(rdev);
2420                 radeon_ib_pool_fini(rdev);
2421                 radeon_vm_manager_fini(rdev);
2422                 radeon_irq_kms_fini(rdev);
2423                 cayman_pcie_gart_fini(rdev);
2424                 rdev->accel_working = false;
2425         }
2426
2427         /* Don't start up if the MC ucode is missing.
2428          * The default clocks and voltages before the MC ucode
2429          * is loaded are not suffient for advanced operations.
2430          *
2431          * We can skip this check for TN, because there is no MC
2432          * ucode.
2433          */
2434         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2435                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2436                 return -EINVAL;
2437         }
2438
2439         return 0;
2440 }
2441
2442 void cayman_fini(struct radeon_device *rdev)
2443 {
2444         radeon_pm_fini(rdev);
2445         cayman_cp_fini(rdev);
2446         cayman_dma_fini(rdev);
2447         r600_irq_fini(rdev);
2448         if (rdev->flags & RADEON_IS_IGP)
2449                 sumo_rlc_fini(rdev);
2450         radeon_wb_fini(rdev);
2451         radeon_vm_manager_fini(rdev);
2452         radeon_ib_pool_fini(rdev);
2453         radeon_irq_kms_fini(rdev);
2454         uvd_v1_0_fini(rdev);
2455         radeon_uvd_fini(rdev);
2456         if (rdev->has_vce)
2457                 radeon_vce_fini(rdev);
2458         cayman_pcie_gart_fini(rdev);
2459         r600_vram_scratch_fini(rdev);
2460         radeon_gem_fini(rdev);
2461         radeon_fence_driver_fini(rdev);
2462         radeon_bo_fini(rdev);
2463         radeon_atombios_fini(rdev);
2464         kfree(rdev->bios);
2465         rdev->bios = NULL;
2466 }
2467
2468 /*
2469  * vm
2470  */
2471 int cayman_vm_init(struct radeon_device *rdev)
2472 {
2473         /* number of VMs */
2474         rdev->vm_manager.nvm = 8;
2475         /* base offset of vram pages */
2476         if (rdev->flags & RADEON_IS_IGP) {
2477                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2478                 tmp <<= 22;
2479                 rdev->vm_manager.vram_base_offset = tmp;
2480         } else
2481                 rdev->vm_manager.vram_base_offset = 0;
2482         return 0;
2483 }
2484
2485 void cayman_vm_fini(struct radeon_device *rdev)
2486 {
2487 }
2488
2489 /**
2490  * cayman_vm_decode_fault - print human readable fault info
2491  *
2492  * @rdev: radeon_device pointer
2493  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2494  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2495  *
2496  * Print human readable fault information (cayman/TN).
2497  */
2498 void cayman_vm_decode_fault(struct radeon_device *rdev,
2499                             u32 status, u32 addr)
2500 {
2501         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2502         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2503         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2504         char *block;
2505
2506         switch (mc_id) {
2507         case 32:
2508         case 16:
2509         case 96:
2510         case 80:
2511         case 160:
2512         case 144:
2513         case 224:
2514         case 208:
2515                 block = "CB";
2516                 break;
2517         case 33:
2518         case 17:
2519         case 97:
2520         case 81:
2521         case 161:
2522         case 145:
2523         case 225:
2524         case 209:
2525                 block = "CB_FMASK";
2526                 break;
2527         case 34:
2528         case 18:
2529         case 98:
2530         case 82:
2531         case 162:
2532         case 146:
2533         case 226:
2534         case 210:
2535                 block = "CB_CMASK";
2536                 break;
2537         case 35:
2538         case 19:
2539         case 99:
2540         case 83:
2541         case 163:
2542         case 147:
2543         case 227:
2544         case 211:
2545                 block = "CB_IMMED";
2546                 break;
2547         case 36:
2548         case 20:
2549         case 100:
2550         case 84:
2551         case 164:
2552         case 148:
2553         case 228:
2554         case 212:
2555                 block = "DB";
2556                 break;
2557         case 37:
2558         case 21:
2559         case 101:
2560         case 85:
2561         case 165:
2562         case 149:
2563         case 229:
2564         case 213:
2565                 block = "DB_HTILE";
2566                 break;
2567         case 38:
2568         case 22:
2569         case 102:
2570         case 86:
2571         case 166:
2572         case 150:
2573         case 230:
2574         case 214:
2575                 block = "SX";
2576                 break;
2577         case 39:
2578         case 23:
2579         case 103:
2580         case 87:
2581         case 167:
2582         case 151:
2583         case 231:
2584         case 215:
2585                 block = "DB_STEN";
2586                 break;
2587         case 40:
2588         case 24:
2589         case 104:
2590         case 88:
2591         case 232:
2592         case 216:
2593         case 168:
2594         case 152:
2595                 block = "TC_TFETCH";
2596                 break;
2597         case 41:
2598         case 25:
2599         case 105:
2600         case 89:
2601         case 233:
2602         case 217:
2603         case 169:
2604         case 153:
2605                 block = "TC_VFETCH";
2606                 break;
2607         case 42:
2608         case 26:
2609         case 106:
2610         case 90:
2611         case 234:
2612         case 218:
2613         case 170:
2614         case 154:
2615                 block = "VC";
2616                 break;
2617         case 112:
2618                 block = "CP";
2619                 break;
2620         case 113:
2621         case 114:
2622                 block = "SH";
2623                 break;
2624         case 115:
2625                 block = "VGT";
2626                 break;
2627         case 178:
2628                 block = "IH";
2629                 break;
2630         case 51:
2631                 block = "RLC";
2632                 break;
2633         case 55:
2634                 block = "DMA";
2635                 break;
2636         case 56:
2637                 block = "HDP";
2638                 break;
2639         default:
2640                 block = "unknown";
2641                 break;
2642         }
2643
2644         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2645                protections, vmid, addr,
2646                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2647                block, mc_id);
2648 }
2649
2650 /*
2651  * cayman_vm_flush - vm flush using the CP
2652  *
2653  * Update the page table base and flush the VM TLB
2654  * using the CP (cayman-si).
2655  */
2656 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2657                      unsigned vm_id, uint64_t pd_addr)
2658 {
2659         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2660         radeon_ring_write(ring, pd_addr >> 12);
2661
2662         /* flush hdp cache */
2663         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2664         radeon_ring_write(ring, 0x1);
2665
2666         /* bits 0-7 are the VM contexts0-7 */
2667         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2668         radeon_ring_write(ring, 1 << vm_id);
2669
2670         /* wait for the invalidate to complete */
2671         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2672         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2673                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2674         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2675         radeon_ring_write(ring, 0);
2676         radeon_ring_write(ring, 0); /* ref */
2677         radeon_ring_write(ring, 0); /* mask */
2678         radeon_ring_write(ring, 0x20); /* poll interval */
2679
2680         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2681         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2682         radeon_ring_write(ring, 0x0);
2683 }
2684
2685 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2686 {
2687         struct atom_clock_dividers dividers;
2688         int r, i;
2689
2690         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2691                                            ecclk, false, &dividers);
2692         if (r)
2693                 return r;
2694
2695         for (i = 0; i < 100; i++) {
2696                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2697                         break;
2698                 mdelay(10);
2699         }
2700         if (i == 100)
2701                 return -ETIMEDOUT;
2702
2703         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2704
2705         for (i = 0; i < 100; i++) {
2706                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2707                         break;
2708                 mdelay(10);
2709         }
2710         if (i == 100)
2711                 return -ETIMEDOUT;
2712
2713         return 0;
2714 }