drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 /*
  40  * Indirect registers accessor
  41  */
  42 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  43 {
  44         unsigned long flags;
  45         u32 r;
  46
  47         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  48         WREG32(TN_SMC_IND_INDEX_0, (reg));
  49         r = RREG32(TN_SMC_IND_DATA_0);
  50         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  51         return r;
  52 }
  53
  54 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  55 {
  56         unsigned long flags;
  57
  58         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  59         WREG32(TN_SMC_IND_INDEX_0, (reg));
  60         WREG32(TN_SMC_IND_DATA_0, (v));
  61         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  62 }
  63
  64 static const u32 tn_rlc_save_restore_register_list[] =
  65 {
  66         0x98fc,
  67         0x98f0,
  68         0x9834,
  69         0x9838,
  70         0x9870,
  71         0x9874,
  72         0x8a14,
  73         0x8b24,
  74         0x8bcc,
  75         0x8b10,
  76         0x8c30,
  77         0x8d00,
  78         0x8d04,
  79         0x8c00,
  80         0x8c04,
  81         0x8c10,
  82         0x8c14,
  83         0x8d8c,
  84         0x8cf0,
  85         0x8e38,
  86         0x9508,
  87         0x9688,
  88         0x9608,
  89         0x960c,
  90         0x9610,
  91         0x9614,
  92         0x88c4,
  93         0x8978,
  94         0x88d4,
  95         0x900c,
  96         0x9100,
  97         0x913c,
  98         0x90e8,
  99         0x9354,
 100         0xa008,
 101         0x98f8,
 102         0x9148,
 103         0x914c,
 104         0x3f94,
 105         0x98f4,
 106         0x9b7c,
 107         0x3f8c,
 108         0x8950,
 109         0x8954,
 110         0x8a18,
 111         0x8b28,
 112         0x9144,
 113         0x3f90,
 114         0x915c,
 115         0x9160,
 116         0x9178,
 117         0x917c,
 118         0x9180,
 119         0x918c,
 120         0x9190,
 121         0x9194,
 122         0x9198,
 123         0x919c,
 124         0x91a8,
 125         0x91ac,
 126         0x91b0,
 127         0x91b4,
 128         0x91b8,
 129         0x91c4,
 130         0x91c8,
 131         0x91cc,
 132         0x91d0,
 133         0x91d4,
 134         0x91e0,
 135         0x91e4,
 136         0x91ec,
 137         0x91f0,
 138         0x91f4,
 139         0x9200,
 140         0x9204,
 141         0x929c,
 142         0x8030,
 143         0x9150,
 144         0x9a60,
 145         0x920c,
 146         0x9210,
 147         0x9228,
 148         0x922c,
 149         0x9244,
 150         0x9248,
 151         0x91e8,
 152         0x9294,
 153         0x9208,
 154         0x9224,
 155         0x9240,
 156         0x9220,
 157         0x923c,
 158         0x9258,
 159         0x9744,
 160         0xa200,
 161         0xa204,
 162         0xa208,
 163         0xa20c,
 164         0x8d58,
 165         0x9030,
 166         0x9034,
 167         0x9038,
 168         0x903c,
 169         0x9040,
 170         0x9654,
 171         0x897c,
 172         0xa210,
 173         0xa214,
 174         0x9868,
 175         0xa02c,
 176         0x9664,
 177         0x9698,
 178         0x949c,
 179         0x8e10,
 180         0x8e18,
 181         0x8c50,
 182         0x8c58,
 183         0x8c60,
 184         0x8c68,
 185         0x89b4,
 186         0x9830,
 187         0x802c,
 188 };
 189
 190 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 191 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 192 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 193 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 194 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 195 extern void evergreen_mc_program(struct radeon_device *rdev);
 196 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 197 extern int evergreen_mc_init(struct radeon_device *rdev);
 198 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 199 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 200 extern void evergreen_program_aspm(struct radeon_device *rdev);
 201 extern void sumo_rlc_fini(struct radeon_device *rdev);
 202 extern int sumo_rlc_init(struct radeon_device *rdev);
 203 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 204
 205 /* Firmware Names */
 206 /*(DEBLOBBED)*/
 207
 208
 209 static const u32 cayman_golden_registers2[] =
 210 {
 211         0x3e5c, 0xffffffff, 0x00000000,
 212         0x3e48, 0xffffffff, 0x00000000,
 213         0x3e4c, 0xffffffff, 0x00000000,
 214         0x3e64, 0xffffffff, 0x00000000,
 215         0x3e50, 0xffffffff, 0x00000000,
 216         0x3e60, 0xffffffff, 0x00000000
 217 };
 218
 219 static const u32 cayman_golden_registers[] =
 220 {
 221         0x5eb4, 0xffffffff, 0x00000002,
 222         0x5e78, 0x8f311ff1, 0x001000f0,
 223         0x3f90, 0xffff0000, 0xff000000,
 224         0x9148, 0xffff0000, 0xff000000,
 225         0x3f94, 0xffff0000, 0xff000000,
 226         0x914c, 0xffff0000, 0xff000000,
 227         0xc78, 0x00000080, 0x00000080,
 228         0xbd4, 0x70073777, 0x00011003,
 229         0xd02c, 0xbfffff1f, 0x08421000,
 230         0xd0b8, 0x73773777, 0x02011003,
 231         0x5bc0, 0x00200000, 0x50100000,
 232         0x98f8, 0x33773777, 0x02011003,
 233         0x98fc, 0xffffffff, 0x76541032,
 234         0x7030, 0x31000311, 0x00000011,
 235         0x2f48, 0x33773777, 0x42010001,
 236         0x6b28, 0x00000010, 0x00000012,
 237         0x7728, 0x00000010, 0x00000012,
 238         0x10328, 0x00000010, 0x00000012,
 239         0x10f28, 0x00000010, 0x00000012,
 240         0x11b28, 0x00000010, 0x00000012,
 241         0x12728, 0x00000010, 0x00000012,
 242         0x240c, 0x000007ff, 0x00000000,
 243         0x8a14, 0xf000001f, 0x00000007,
 244         0x8b24, 0x3fff3fff, 0x00ff0fff,
 245         0x8b10, 0x0000ff0f, 0x00000000,
 246         0x28a4c, 0x07ffffff, 0x06000000,
 247         0x10c, 0x00000001, 0x00010003,
 248         0xa02c, 0xffffffff, 0x0000009b,
 249         0x913c, 0x0000010f, 0x01000100,
 250         0x8c04, 0xf8ff00ff, 0x40600060,
 251         0x28350, 0x00000f01, 0x00000000,
 252         0x9508, 0x3700001f, 0x00000002,
 253         0x960c, 0xffffffff, 0x54763210,
 254         0x88c4, 0x001f3ae3, 0x00000082,
 255         0x88d0, 0xffffffff, 0x0f40df40,
 256         0x88d4, 0x0000001f, 0x00000010,
 257         0x8974, 0xffffffff, 0x00000000
 258 };
 259
 260 static const u32 dvst_golden_registers2[] =
 261 {
 262         0x8f8, 0xffffffff, 0,
 263         0x8fc, 0x00380000, 0,
 264         0x8f8, 0xffffffff, 1,
 265         0x8fc, 0x0e000000, 0
 266 };
 267
 268 static const u32 dvst_golden_registers[] =
 269 {
 270         0x690, 0x3fff3fff, 0x20c00033,
 271         0x918c, 0x0fff0fff, 0x00010006,
 272         0x91a8, 0x0fff0fff, 0x00010006,
 273         0x9150, 0xffffdfff, 0x6e944040,
 274         0x917c, 0x0fff0fff, 0x00030002,
 275         0x9198, 0x0fff0fff, 0x00030002,
 276         0x915c, 0x0fff0fff, 0x00010000,
 277         0x3f90, 0xffff0001, 0xff000000,
 278         0x9178, 0x0fff0fff, 0x00070000,
 279         0x9194, 0x0fff0fff, 0x00070000,
 280         0x9148, 0xffff0001, 0xff000000,
 281         0x9190, 0x0fff0fff, 0x00090008,
 282         0x91ac, 0x0fff0fff, 0x00090008,
 283         0x3f94, 0xffff0000, 0xff000000,
 284         0x914c, 0xffff0000, 0xff000000,
 285         0x929c, 0x00000fff, 0x00000001,
 286         0x55e4, 0xff607fff, 0xfc000100,
 287         0x8a18, 0xff000fff, 0x00000100,
 288         0x8b28, 0xff000fff, 0x00000100,
 289         0x9144, 0xfffc0fff, 0x00000100,
 290         0x6ed8, 0x00010101, 0x00010000,
 291         0x9830, 0xffffffff, 0x00000000,
 292         0x9834, 0xf00fffff, 0x00000400,
 293         0x9838, 0xfffffffe, 0x00000000,
 294         0xd0c0, 0xff000fff, 0x00000100,
 295         0xd02c, 0xbfffff1f, 0x08421000,
 296         0xd0b8, 0x73773777, 0x12010001,
 297         0x5bb0, 0x000000f0, 0x00000070,
 298         0x98f8, 0x73773777, 0x12010001,
 299         0x98fc, 0xffffffff, 0x00000010,
 300         0x9b7c, 0x00ff0000, 0x00fc0000,
 301         0x8030, 0x00001f0f, 0x0000100a,
 302         0x2f48, 0x73773777, 0x12010001,
 303         0x2408, 0x00030000, 0x000c007f,
 304         0x8a14, 0xf000003f, 0x00000007,
 305         0x8b24, 0x3fff3fff, 0x00ff0fff,
 306         0x8b10, 0x0000ff0f, 0x00000000,
 307         0x28a4c, 0x07ffffff, 0x06000000,
 308         0x4d8, 0x00000fff, 0x00000100,
 309         0xa008, 0xffffffff, 0x00010000,
 310         0x913c, 0xffff03ff, 0x01000100,
 311         0x8c00, 0x000000ff, 0x00000003,
 312         0x8c04, 0xf8ff00ff, 0x40600060,
 313         0x8cf0, 0x1fff1fff, 0x08e00410,
 314         0x28350, 0x00000f01, 0x00000000,
 315         0x9508, 0xf700071f, 0x00000002,
 316         0x960c, 0xffffffff, 0x54763210,
 317         0x20ef8, 0x01ff01ff, 0x00000002,
 318         0x20e98, 0xfffffbff, 0x00200000,
 319         0x2015c, 0xffffffff, 0x00000f40,
 320         0x88c4, 0x001f3ae3, 0x00000082,
 321         0x8978, 0x3fffffff, 0x04050140,
 322         0x88d4, 0x0000001f, 0x00000010,
 323         0x8974, 0xffffffff, 0x00000000
 324 };
 325
 326 static const u32 scrapper_golden_registers[] =
 327 {
 328         0x690, 0x3fff3fff, 0x20c00033,
 329         0x918c, 0x0fff0fff, 0x00010006,
 330         0x918c, 0x0fff0fff, 0x00010006,
 331         0x91a8, 0x0fff0fff, 0x00010006,
 332         0x91a8, 0x0fff0fff, 0x00010006,
 333         0x9150, 0xffffdfff, 0x6e944040,
 334         0x9150, 0xffffdfff, 0x6e944040,
 335         0x917c, 0x0fff0fff, 0x00030002,
 336         0x917c, 0x0fff0fff, 0x00030002,
 337         0x9198, 0x0fff0fff, 0x00030002,
 338         0x9198, 0x0fff0fff, 0x00030002,
 339         0x915c, 0x0fff0fff, 0x00010000,
 340         0x915c, 0x0fff0fff, 0x00010000,
 341         0x3f90, 0xffff0001, 0xff000000,
 342         0x3f90, 0xffff0001, 0xff000000,
 343         0x9178, 0x0fff0fff, 0x00070000,
 344         0x9178, 0x0fff0fff, 0x00070000,
 345         0x9194, 0x0fff0fff, 0x00070000,
 346         0x9194, 0x0fff0fff, 0x00070000,
 347         0x9148, 0xffff0001, 0xff000000,
 348         0x9148, 0xffff0001, 0xff000000,
 349         0x9190, 0x0fff0fff, 0x00090008,
 350         0x9190, 0x0fff0fff, 0x00090008,
 351         0x91ac, 0x0fff0fff, 0x00090008,
 352         0x91ac, 0x0fff0fff, 0x00090008,
 353         0x3f94, 0xffff0000, 0xff000000,
 354         0x3f94, 0xffff0000, 0xff000000,
 355         0x914c, 0xffff0000, 0xff000000,
 356         0x914c, 0xffff0000, 0xff000000,
 357         0x929c, 0x00000fff, 0x00000001,
 358         0x929c, 0x00000fff, 0x00000001,
 359         0x55e4, 0xff607fff, 0xfc000100,
 360         0x8a18, 0xff000fff, 0x00000100,
 361         0x8a18, 0xff000fff, 0x00000100,
 362         0x8b28, 0xff000fff, 0x00000100,
 363         0x8b28, 0xff000fff, 0x00000100,
 364         0x9144, 0xfffc0fff, 0x00000100,
 365         0x9144, 0xfffc0fff, 0x00000100,
 366         0x6ed8, 0x00010101, 0x00010000,
 367         0x9830, 0xffffffff, 0x00000000,
 368         0x9830, 0xffffffff, 0x00000000,
 369         0x9834, 0xf00fffff, 0x00000400,
 370         0x9834, 0xf00fffff, 0x00000400,
 371         0x9838, 0xfffffffe, 0x00000000,
 372         0x9838, 0xfffffffe, 0x00000000,
 373         0xd0c0, 0xff000fff, 0x00000100,
 374         0xd02c, 0xbfffff1f, 0x08421000,
 375         0xd02c, 0xbfffff1f, 0x08421000,
 376         0xd0b8, 0x73773777, 0x12010001,
 377         0xd0b8, 0x73773777, 0x12010001,
 378         0x5bb0, 0x000000f0, 0x00000070,
 379         0x98f8, 0x73773777, 0x12010001,
 380         0x98f8, 0x73773777, 0x12010001,
 381         0x98fc, 0xffffffff, 0x00000010,
 382         0x98fc, 0xffffffff, 0x00000010,
 383         0x9b7c, 0x00ff0000, 0x00fc0000,
 384         0x9b7c, 0x00ff0000, 0x00fc0000,
 385         0x8030, 0x00001f0f, 0x0000100a,
 386         0x8030, 0x00001f0f, 0x0000100a,
 387         0x2f48, 0x73773777, 0x12010001,
 388         0x2f48, 0x73773777, 0x12010001,
 389         0x2408, 0x00030000, 0x000c007f,
 390         0x8a14, 0xf000003f, 0x00000007,
 391         0x8a14, 0xf000003f, 0x00000007,
 392         0x8b24, 0x3fff3fff, 0x00ff0fff,
 393         0x8b24, 0x3fff3fff, 0x00ff0fff,
 394         0x8b10, 0x0000ff0f, 0x00000000,
 395         0x8b10, 0x0000ff0f, 0x00000000,
 396         0x28a4c, 0x07ffffff, 0x06000000,
 397         0x28a4c, 0x07ffffff, 0x06000000,
 398         0x4d8, 0x00000fff, 0x00000100,
 399         0x4d8, 0x00000fff, 0x00000100,
 400         0xa008, 0xffffffff, 0x00010000,
 401         0xa008, 0xffffffff, 0x00010000,
 402         0x913c, 0xffff03ff, 0x01000100,
 403         0x913c, 0xffff03ff, 0x01000100,
 404         0x90e8, 0x001fffff, 0x010400c0,
 405         0x8c00, 0x000000ff, 0x00000003,
 406         0x8c00, 0x000000ff, 0x00000003,
 407         0x8c04, 0xf8ff00ff, 0x40600060,
 408         0x8c04, 0xf8ff00ff, 0x40600060,
 409         0x8c30, 0x0000000f, 0x00040005,
 410         0x8cf0, 0x1fff1fff, 0x08e00410,
 411         0x8cf0, 0x1fff1fff, 0x08e00410,
 412         0x900c, 0x00ffffff, 0x0017071f,
 413         0x28350, 0x00000f01, 0x00000000,
 414         0x28350, 0x00000f01, 0x00000000,
 415         0x9508, 0xf700071f, 0x00000002,
 416         0x9508, 0xf700071f, 0x00000002,
 417         0x9688, 0x00300000, 0x0017000f,
 418         0x960c, 0xffffffff, 0x54763210,
 419         0x960c, 0xffffffff, 0x54763210,
 420         0x20ef8, 0x01ff01ff, 0x00000002,
 421         0x20e98, 0xfffffbff, 0x00200000,
 422         0x2015c, 0xffffffff, 0x00000f40,
 423         0x88c4, 0x001f3ae3, 0x00000082,
 424         0x88c4, 0x001f3ae3, 0x00000082,
 425         0x8978, 0x3fffffff, 0x04050140,
 426         0x8978, 0x3fffffff, 0x04050140,
 427         0x88d4, 0x0000001f, 0x00000010,
 428         0x88d4, 0x0000001f, 0x00000010,
 429         0x8974, 0xffffffff, 0x00000000,
 430         0x8974, 0xffffffff, 0x00000000
 431 };
 432
 433 static void ni_init_golden_registers(struct radeon_device *rdev)
 434 {
 435         switch (rdev->family) {
 436         case CHIP_CAYMAN:
 437                 radeon_program_register_sequence(rdev,
 438                                                  cayman_golden_registers,
 439                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 440                 radeon_program_register_sequence(rdev,
 441                                                  cayman_golden_registers2,
 442                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 443                 break;
 444         case CHIP_ARUBA:
 445                 if ((rdev->pdev->device == 0x9900) ||
 446                     (rdev->pdev->device == 0x9901) ||
 447                     (rdev->pdev->device == 0x9903) ||
 448                     (rdev->pdev->device == 0x9904) ||
 449                     (rdev->pdev->device == 0x9905) ||
 450                     (rdev->pdev->device == 0x9906) ||
 451                     (rdev->pdev->device == 0x9907) ||
 452                     (rdev->pdev->device == 0x9908) ||
 453                     (rdev->pdev->device == 0x9909) ||
 454                     (rdev->pdev->device == 0x990A) ||
 455                     (rdev->pdev->device == 0x990B) ||
 456                     (rdev->pdev->device == 0x990C) ||
 457                     (rdev->pdev->device == 0x990D) ||
 458                     (rdev->pdev->device == 0x990E) ||
 459                     (rdev->pdev->device == 0x990F) ||
 460                     (rdev->pdev->device == 0x9910) ||
 461                     (rdev->pdev->device == 0x9913) ||
 462                     (rdev->pdev->device == 0x9917) ||
 463                     (rdev->pdev->device == 0x9918)) {
 464                         radeon_program_register_sequence(rdev,
 465                                                          dvst_golden_registers,
 466                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 467                         radeon_program_register_sequence(rdev,
 468                                                          dvst_golden_registers2,
 469                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 470                 } else {
 471                         radeon_program_register_sequence(rdev,
 472                                                          scrapper_golden_registers,
 473                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 474                         radeon_program_register_sequence(rdev,
 475                                                          dvst_golden_registers2,
 476                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 477                 }
 478                 break;
 479         default:
 480                 break;
 481         }
 482 }
 483
 484 #define BTC_IO_MC_REGS_SIZE 29
 485
 486 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 487         {0x00000077, 0xff010100},
 488         {0x00000078, 0x00000000},
 489         {0x00000079, 0x00001434},
 490         {0x0000007a, 0xcc08ec08},
 491         {0x0000007b, 0x00040000},
 492         {0x0000007c, 0x000080c0},
 493         {0x0000007d, 0x09000000},
 494         {0x0000007e, 0x00210404},
 495         {0x00000081, 0x08a8e800},
 496         {0x00000082, 0x00030444},
 497         {0x00000083, 0x00000000},
 498         {0x00000085, 0x00000001},
 499         {0x00000086, 0x00000002},
 500         {0x00000087, 0x48490000},
 501         {0x00000088, 0x20244647},
 502         {0x00000089, 0x00000005},
 503         {0x0000008b, 0x66030000},
 504         {0x0000008c, 0x00006603},
 505         {0x0000008d, 0x00000100},
 506         {0x0000008f, 0x00001c0a},
 507         {0x00000090, 0xff000001},
 508         {0x00000094, 0x00101101},
 509         {0x00000095, 0x00000fff},
 510         {0x00000096, 0x00116fff},
 511         {0x00000097, 0x60010000},
 512         {0x00000098, 0x10010000},
 513         {0x00000099, 0x00006000},
 514         {0x0000009a, 0x00001000},
 515         {0x0000009f, 0x00946a00}
 516 };
 517
 518 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 519         {0x00000077, 0xff010100},
 520         {0x00000078, 0x00000000},
 521         {0x00000079, 0x00001434},
 522         {0x0000007a, 0xcc08ec08},
 523         {0x0000007b, 0x00040000},
 524         {0x0000007c, 0x000080c0},
 525         {0x0000007d, 0x09000000},
 526         {0x0000007e, 0x00210404},
 527         {0x00000081, 0x08a8e800},
 528         {0x00000082, 0x00030444},
 529         {0x00000083, 0x00000000},
 530         {0x00000085, 0x00000001},
 531         {0x00000086, 0x00000002},
 532         {0x00000087, 0x48490000},
 533         {0x00000088, 0x20244647},
 534         {0x00000089, 0x00000005},
 535         {0x0000008b, 0x66030000},
 536         {0x0000008c, 0x00006603},
 537         {0x0000008d, 0x00000100},
 538         {0x0000008f, 0x00001c0a},
 539         {0x00000090, 0xff000001},
 540         {0x00000094, 0x00101101},
 541         {0x00000095, 0x00000fff},
 542         {0x00000096, 0x00116fff},
 543         {0x00000097, 0x60010000},
 544         {0x00000098, 0x10010000},
 545         {0x00000099, 0x00006000},
 546         {0x0000009a, 0x00001000},
 547         {0x0000009f, 0x00936a00}
 548 };
 549
 550 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 551         {0x00000077, 0xff010100},
 552         {0x00000078, 0x00000000},
 553         {0x00000079, 0x00001434},
 554         {0x0000007a, 0xcc08ec08},
 555         {0x0000007b, 0x00040000},
 556         {0x0000007c, 0x000080c0},
 557         {0x0000007d, 0x09000000},
 558         {0x0000007e, 0x00210404},
 559         {0x00000081, 0x08a8e800},
 560         {0x00000082, 0x00030444},
 561         {0x00000083, 0x00000000},
 562         {0x00000085, 0x00000001},
 563         {0x00000086, 0x00000002},
 564         {0x00000087, 0x48490000},
 565         {0x00000088, 0x20244647},
 566         {0x00000089, 0x00000005},
 567         {0x0000008b, 0x66030000},
 568         {0x0000008c, 0x00006603},
 569         {0x0000008d, 0x00000100},
 570         {0x0000008f, 0x00001c0a},
 571         {0x00000090, 0xff000001},
 572         {0x00000094, 0x00101101},
 573         {0x00000095, 0x00000fff},
 574         {0x00000096, 0x00116fff},
 575         {0x00000097, 0x60010000},
 576         {0x00000098, 0x10010000},
 577         {0x00000099, 0x00006000},
 578         {0x0000009a, 0x00001000},
 579         {0x0000009f, 0x00916a00}
 580 };
 581
 582 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 583         {0x00000077, 0xff010100},
 584         {0x00000078, 0x00000000},
 585         {0x00000079, 0x00001434},
 586         {0x0000007a, 0xcc08ec08},
 587         {0x0000007b, 0x00040000},
 588         {0x0000007c, 0x000080c0},
 589         {0x0000007d, 0x09000000},
 590         {0x0000007e, 0x00210404},
 591         {0x00000081, 0x08a8e800},
 592         {0x00000082, 0x00030444},
 593         {0x00000083, 0x00000000},
 594         {0x00000085, 0x00000001},
 595         {0x00000086, 0x00000002},
 596         {0x00000087, 0x48490000},
 597         {0x00000088, 0x20244647},
 598         {0x00000089, 0x00000005},
 599         {0x0000008b, 0x66030000},
 600         {0x0000008c, 0x00006603},
 601         {0x0000008d, 0x00000100},
 602         {0x0000008f, 0x00001c0a},
 603         {0x00000090, 0xff000001},
 604         {0x00000094, 0x00101101},
 605         {0x00000095, 0x00000fff},
 606         {0x00000096, 0x00116fff},
 607         {0x00000097, 0x60010000},
 608         {0x00000098, 0x10010000},
 609         {0x00000099, 0x00006000},
 610         {0x0000009a, 0x00001000},
 611         {0x0000009f, 0x00976b00}
 612 };
 613
 614 int ni_mc_load_microcode(struct radeon_device *rdev)
 615 {
 616         const __be32 *fw_data;
 617         u32 mem_type, running, blackout = 0;
 618         u32 *io_mc_regs;
 619         int i, ucode_size, regs_size;
 620
 621         if (!rdev->mc_fw)
 622                 return -EINVAL;
 623
 624         switch (rdev->family) {
 625         case CHIP_BARTS:
 626                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 627                 ucode_size = BTC_MC_UCODE_SIZE;
 628                 regs_size = BTC_IO_MC_REGS_SIZE;
 629                 break;
 630         case CHIP_TURKS:
 631                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 632                 ucode_size = BTC_MC_UCODE_SIZE;
 633                 regs_size = BTC_IO_MC_REGS_SIZE;
 634                 break;
 635         case CHIP_CAICOS:
 636         default:
 637                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 638                 ucode_size = BTC_MC_UCODE_SIZE;
 639                 regs_size = BTC_IO_MC_REGS_SIZE;
 640                 break;
 641         case CHIP_CAYMAN:
 642                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 643                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 644                 regs_size = BTC_IO_MC_REGS_SIZE;
 645                 break;
 646         }
 647
 648         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 649         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 650
 651         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 652                 if (running) {
 653                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 654                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 655                 }
 656
 657                 /* reset the engine and set to writable */
 658                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 659                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 660
 661                 /* load mc io regs */
 662                 for (i = 0; i < regs_size; i++) {
 663                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 664                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 665                 }
 666                 /* load the MC ucode */
 667                 fw_data = (const __be32 *)rdev->mc_fw->data;
 668                 for (i = 0; i < ucode_size; i++)
 669                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 670
 671                 /* put the engine back into the active state */
 672                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 673                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 674                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 675
 676                 /* wait for training to complete */
 677                 for (i = 0; i < rdev->usec_timeout; i++) {
 678                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 679                                 break;
 680                         udelay(1);
 681                 }
 682
 683                 if (running)
 684                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 685         }
 686
 687         return 0;
 688 }
 689
 690 int ni_init_microcode(struct radeon_device *rdev)
 691 {
 692         const char *chip_name;
 693         const char *rlc_chip_name;
 694         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 695         size_t smc_req_size = 0;
 696         char fw_name[30];
 697         int err;
 698
 699         DRM_DEBUG("\n");
 700
 701         switch (rdev->family) {
 702         case CHIP_BARTS:
 703                 chip_name = "BARTS";
 704                 rlc_chip_name = "BTC";
 705                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 706                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 707                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 708                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 709                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 710                 break;
 711         case CHIP_TURKS:
 712                 chip_name = "TURKS";
 713                 rlc_chip_name = "BTC";
 714                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 715                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 716                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 717                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 718                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 719                 break;
 720         case CHIP_CAICOS:
 721                 chip_name = "CAICOS";
 722                 rlc_chip_name = "BTC";
 723                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 724                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 725                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 726                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 727                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 728                 break;
 729         case CHIP_CAYMAN:
 730                 chip_name = "CAYMAN";
 731                 rlc_chip_name = "CAYMAN";
 732                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 733                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 734                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 735                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 736                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 737                 break;
 738         case CHIP_ARUBA:
 739                 chip_name = "ARUBA";
 740                 rlc_chip_name = "ARUBA";
 741                 /* pfp/me same size as CAYMAN */
 742                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 743                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 744                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 745                 mc_req_size = 0;
 746                 break;
 747         default: BUG();
 748         }
 749
 750         DRM_INFO("Loading %s Microcode\n", chip_name);
 751
 752         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 753         err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 754         if (err)
 755                 goto out;
 756         if (rdev->pfp_fw->size != pfp_req_size) {
 757                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 758                        rdev->pfp_fw->size, fw_name);
 759                 err = -EINVAL;
 760                 goto out;
 761         }
 762
 763         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 764         err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
 765         if (err)
 766                 goto out;
 767         if (rdev->me_fw->size != me_req_size) {
 768                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 769                        rdev->me_fw->size, fw_name);
 770                 err = -EINVAL;
 771         }
 772
 773         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", rlc_chip_name);
 774         err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 775         if (err)
 776                 goto out;
 777         if (rdev->rlc_fw->size != rlc_req_size) {
 778                 pr_err("ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 779                        rdev->rlc_fw->size, fw_name);
 780                 err = -EINVAL;
 781         }
 782
 783         /* no MC ucode on TN */
 784         if (!(rdev->flags & RADEON_IS_IGP)) {
 785                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 786                 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 787                 if (err)
 788                         goto out;
 789                 if (rdev->mc_fw->size != mc_req_size) {
 790                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 791                                rdev->mc_fw->size, fw_name);
 792                         err = -EINVAL;
 793                 }
 794         }
 795
 796         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 797                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 798                 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 799                 if (err) {
 800                         pr_err("smc: error loading firmware \"%s\"\n", fw_name);
 801                         release_firmware(rdev->smc_fw);
 802                         rdev->smc_fw = NULL;
 803                         err = 0;
 804                 } else if (rdev->smc_fw->size != smc_req_size) {
 805                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 806                                rdev->mc_fw->size, fw_name);
 807                         err = -EINVAL;
 808                 }
 809         }
 810
 811 out:
 812         if (err) {
 813                 if (err != -EINVAL)
 814                         pr_err("ni_cp: Failed to load firmware \"%s\"\n",
 815                                fw_name);
 816                 release_firmware(rdev->pfp_fw);
 817                 rdev->pfp_fw = NULL;
 818                 release_firmware(rdev->me_fw);
 819                 rdev->me_fw = NULL;
 820                 release_firmware(rdev->rlc_fw);
 821                 rdev->rlc_fw = NULL;
 822                 release_firmware(rdev->mc_fw);
 823                 rdev->mc_fw = NULL;
 824         }
 825         return err;
 826 }
 827
 828 /**
 829  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 830  *
 831  * @rdev: radeon_device pointer
 832  * @reg: register offset in bytes
 833  * @val: register value
 834  *
 835  * Returns 0 for success or -EINVAL for an invalid register
 836  *
 837  */
 838 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 839                                      u32 reg, u32 *val)
 840 {
 841         switch (reg) {
 842         case GRBM_STATUS:
 843         case GRBM_STATUS_SE0:
 844         case GRBM_STATUS_SE1:
 845         case SRBM_STATUS:
 846         case SRBM_STATUS2:
 847         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 848         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 849         case UVD_STATUS:
 850                 *val = RREG32(reg);
 851                 return 0;
 852         default:
 853                 return -EINVAL;
 854         }
 855 }
 856
 857 int tn_get_temp(struct radeon_device *rdev)
 858 {
 859         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 860         int actual_temp = (temp / 8) - 49;
 861
 862         return actual_temp * 1000;
 863 }
 864
 865 /*
 866  * Core functions
 867  */
 868 static void cayman_gpu_init(struct radeon_device *rdev)
 869 {
 870         u32 gb_addr_config = 0;
 871         u32 mc_shared_chmap, mc_arb_ramcfg;
 872         u32 cgts_tcc_disable;
 873         u32 sx_debug_1;
 874         u32 smx_dc_ctl0;
 875         u32 cgts_sm_ctrl_reg;
 876         u32 hdp_host_path_cntl;
 877         u32 tmp;
 878         u32 disabled_rb_mask;
 879         int i, j;
 880
 881         switch (rdev->family) {
 882         case CHIP_CAYMAN:
 883                 rdev->config.cayman.max_shader_engines = 2;
 884                 rdev->config.cayman.max_pipes_per_simd = 4;
 885                 rdev->config.cayman.max_tile_pipes = 8;
 886                 rdev->config.cayman.max_simds_per_se = 12;
 887                 rdev->config.cayman.max_backends_per_se = 4;
 888                 rdev->config.cayman.max_texture_channel_caches = 8;
 889                 rdev->config.cayman.max_gprs = 256;
 890                 rdev->config.cayman.max_threads = 256;
 891                 rdev->config.cayman.max_gs_threads = 32;
 892                 rdev->config.cayman.max_stack_entries = 512;
 893                 rdev->config.cayman.sx_num_of_sets = 8;
 894                 rdev->config.cayman.sx_max_export_size = 256;
 895                 rdev->config.cayman.sx_max_export_pos_size = 64;
 896                 rdev->config.cayman.sx_max_export_smx_size = 192;
 897                 rdev->config.cayman.max_hw_contexts = 8;
 898                 rdev->config.cayman.sq_num_cf_insts = 2;
 899
 900                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 901                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 902                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 903                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 904                 break;
 905         case CHIP_ARUBA:
 906         default:
 907                 rdev->config.cayman.max_shader_engines = 1;
 908                 rdev->config.cayman.max_pipes_per_simd = 4;
 909                 rdev->config.cayman.max_tile_pipes = 2;
 910                 if ((rdev->pdev->device == 0x9900) ||
 911                     (rdev->pdev->device == 0x9901) ||
 912                     (rdev->pdev->device == 0x9905) ||
 913                     (rdev->pdev->device == 0x9906) ||
 914                     (rdev->pdev->device == 0x9907) ||
 915                     (rdev->pdev->device == 0x9908) ||
 916                     (rdev->pdev->device == 0x9909) ||
 917                     (rdev->pdev->device == 0x990B) ||
 918                     (rdev->pdev->device == 0x990C) ||
 919                     (rdev->pdev->device == 0x990F) ||
 920                     (rdev->pdev->device == 0x9910) ||
 921                     (rdev->pdev->device == 0x9917) ||
 922                     (rdev->pdev->device == 0x9999) ||
 923                     (rdev->pdev->device == 0x999C)) {
 924                         rdev->config.cayman.max_simds_per_se = 6;
 925                         rdev->config.cayman.max_backends_per_se = 2;
 926                         rdev->config.cayman.max_hw_contexts = 8;
 927                         rdev->config.cayman.sx_max_export_size = 256;
 928                         rdev->config.cayman.sx_max_export_pos_size = 64;
 929                         rdev->config.cayman.sx_max_export_smx_size = 192;
 930                 } else if ((rdev->pdev->device == 0x9903) ||
 931                            (rdev->pdev->device == 0x9904) ||
 932                            (rdev->pdev->device == 0x990A) ||
 933                            (rdev->pdev->device == 0x990D) ||
 934                            (rdev->pdev->device == 0x990E) ||
 935                            (rdev->pdev->device == 0x9913) ||
 936                            (rdev->pdev->device == 0x9918) ||
 937                            (rdev->pdev->device == 0x999D)) {
 938                         rdev->config.cayman.max_simds_per_se = 4;
 939                         rdev->config.cayman.max_backends_per_se = 2;
 940                         rdev->config.cayman.max_hw_contexts = 8;
 941                         rdev->config.cayman.sx_max_export_size = 256;
 942                         rdev->config.cayman.sx_max_export_pos_size = 64;
 943                         rdev->config.cayman.sx_max_export_smx_size = 192;
 944                 } else if ((rdev->pdev->device == 0x9919) ||
 945                            (rdev->pdev->device == 0x9990) ||
 946                            (rdev->pdev->device == 0x9991) ||
 947                            (rdev->pdev->device == 0x9994) ||
 948                            (rdev->pdev->device == 0x9995) ||
 949                            (rdev->pdev->device == 0x9996) ||
 950                            (rdev->pdev->device == 0x999A) ||
 951                            (rdev->pdev->device == 0x99A0)) {
 952                         rdev->config.cayman.max_simds_per_se = 3;
 953                         rdev->config.cayman.max_backends_per_se = 1;
 954                         rdev->config.cayman.max_hw_contexts = 4;
 955                         rdev->config.cayman.sx_max_export_size = 128;
 956                         rdev->config.cayman.sx_max_export_pos_size = 32;
 957                         rdev->config.cayman.sx_max_export_smx_size = 96;
 958                 } else {
 959                         rdev->config.cayman.max_simds_per_se = 2;
 960                         rdev->config.cayman.max_backends_per_se = 1;
 961                         rdev->config.cayman.max_hw_contexts = 4;
 962                         rdev->config.cayman.sx_max_export_size = 128;
 963                         rdev->config.cayman.sx_max_export_pos_size = 32;
 964                         rdev->config.cayman.sx_max_export_smx_size = 96;
 965                 }
 966                 rdev->config.cayman.max_texture_channel_caches = 2;
 967                 rdev->config.cayman.max_gprs = 256;
 968                 rdev->config.cayman.max_threads = 256;
 969                 rdev->config.cayman.max_gs_threads = 32;
 970                 rdev->config.cayman.max_stack_entries = 512;
 971                 rdev->config.cayman.sx_num_of_sets = 8;
 972                 rdev->config.cayman.sq_num_cf_insts = 2;
 973
 974                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 975                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 976                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 977                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 978                 break;
 979         }
 980
 981         /* Initialize HDP */
 982         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 983                 WREG32((0x2c14 + j), 0x00000000);
 984                 WREG32((0x2c18 + j), 0x00000000);
 985                 WREG32((0x2c1c + j), 0x00000000);
 986                 WREG32((0x2c20 + j), 0x00000000);
 987                 WREG32((0x2c24 + j), 0x00000000);
 988         }
 989
 990         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 991         WREG32(SRBM_INT_CNTL, 0x1);
 992         WREG32(SRBM_INT_ACK, 0x1);
 993
 994         evergreen_fix_pci_max_read_req_size(rdev);
 995
 996         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 997         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 998
 999         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1000         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1001         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1002                 rdev->config.cayman.mem_row_size_in_kb = 4;
1003         /* XXX use MC settings? */
1004         rdev->config.cayman.shader_engine_tile_size = 32;
1005         rdev->config.cayman.num_gpus = 1;
1006         rdev->config.cayman.multi_gpu_tile_size = 64;
1007
1008         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1009         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1010         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1011         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1012         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1013         rdev->config.cayman.num_shader_engines = tmp + 1;
1014         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1015         rdev->config.cayman.num_gpus = tmp + 1;
1016         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1017         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1018         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1019         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1020
1021
1022         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1023          * not have bank info, so create a custom tiling dword.
1024          * bits 3:0   num_pipes
1025          * bits 7:4   num_banks
1026          * bits 11:8  group_size
1027          * bits 15:12 row_size
1028          */
1029         rdev->config.cayman.tile_config = 0;
1030         switch (rdev->config.cayman.num_tile_pipes) {
1031         case 1:
1032         default:
1033                 rdev->config.cayman.tile_config |= (0 << 0);
1034                 break;
1035         case 2:
1036                 rdev->config.cayman.tile_config |= (1 << 0);
1037                 break;
1038         case 4:
1039                 rdev->config.cayman.tile_config |= (2 << 0);
1040                 break;
1041         case 8:
1042                 rdev->config.cayman.tile_config |= (3 << 0);
1043                 break;
1044         }
1045
1046         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1047         if (rdev->flags & RADEON_IS_IGP)
1048                 rdev->config.cayman.tile_config |= 1 << 4;
1049         else {
1050                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1051                 case 0: /* four banks */
1052                         rdev->config.cayman.tile_config |= 0 << 4;
1053                         break;
1054                 case 1: /* eight banks */
1055                         rdev->config.cayman.tile_config |= 1 << 4;
1056                         break;
1057                 case 2: /* sixteen banks */
1058                 default:
1059                         rdev->config.cayman.tile_config |= 2 << 4;
1060                         break;
1061                 }
1062         }
1063         rdev->config.cayman.tile_config |=
1064                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1065         rdev->config.cayman.tile_config |=
1066                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1067
1068         tmp = 0;
1069         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1070                 u32 rb_disable_bitmap;
1071
1072                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1073                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1074                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1075                 tmp <<= 4;
1076                 tmp |= rb_disable_bitmap;
1077         }
1078         /* enabled rb are just the one not disabled :) */
1079         disabled_rb_mask = tmp;
1080         tmp = 0;
1081         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1082                 tmp |= (1 << i);
1083         /* if all the backends are disabled, fix it up here */
1084         if ((disabled_rb_mask & tmp) == tmp) {
1085                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1086                         disabled_rb_mask &= ~(1 << i);
1087         }
1088
1089         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1090                 u32 simd_disable_bitmap;
1091
1092                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1093                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1094                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1095                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1096                 tmp <<= 16;
1097                 tmp |= simd_disable_bitmap;
1098         }
1099         rdev->config.cayman.active_simds = hweight32(~tmp);
1100
1101         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1102         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1103
1104         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1105         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1106         if (ASIC_IS_DCE6(rdev))
1107                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1108         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1109         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1110         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1111         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1112         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1113         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1114
1115         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1116             (rdev->flags & RADEON_IS_IGP)) {
1117                 if ((disabled_rb_mask & 3) == 2) {
1118                         /* RB1 disabled, RB0 enabled */
1119                         tmp = 0x00000000;
1120                 } else {
1121                         /* RB0 disabled, RB1 enabled */
1122                         tmp = 0x11111111;
1123                 }
1124         } else {
1125                 tmp = gb_addr_config & NUM_PIPES_MASK;
1126                 tmp = r6xx_remap_render_backend(rdev, tmp,
1127                                                 rdev->config.cayman.max_backends_per_se *
1128                                                 rdev->config.cayman.max_shader_engines,
1129                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1130         }
1131         rdev->config.cayman.backend_map = tmp;
1132         WREG32(GB_BACKEND_MAP, tmp);
1133
1134         cgts_tcc_disable = 0xffff0000;
1135         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1136                 cgts_tcc_disable &= ~(1 << (16 + i));
1137         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1138         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1139         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1140         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1141
1142         /* reprogram the shader complex */
1143         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1144         for (i = 0; i < 16; i++)
1145                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1146         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1147
1148         /* set HW defaults for 3D engine */
1149         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1150
1151         sx_debug_1 = RREG32(SX_DEBUG_1);
1152         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1153         WREG32(SX_DEBUG_1, sx_debug_1);
1154
1155         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1156         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1157         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1158         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1159
1160         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1161
1162         /* need to be explicitly zero-ed */
1163         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1164         WREG32(SQ_LSTMP_RING_BASE, 0);
1165         WREG32(SQ_HSTMP_RING_BASE, 0);
1166         WREG32(SQ_ESTMP_RING_BASE, 0);
1167         WREG32(SQ_GSTMP_RING_BASE, 0);
1168         WREG32(SQ_VSTMP_RING_BASE, 0);
1169         WREG32(SQ_PSTMP_RING_BASE, 0);
1170
1171         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1172
1173         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1174                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1175                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1176
1177         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1178                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1179                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1180
1181
1182         WREG32(VGT_NUM_INSTANCES, 1);
1183
1184         WREG32(CP_PERFMON_CNTL, 0);
1185
1186         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1187                                   FETCH_FIFO_HIWATER(0x4) |
1188                                   DONE_FIFO_HIWATER(0xe0) |
1189                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1190
1191         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1192         WREG32(SQ_CONFIG, (VC_ENABLE |
1193                            EXPORT_SRC_C |
1194                            GFX_PRIO(0) |
1195                            CS1_PRIO(0) |
1196                            CS2_PRIO(1)));
1197         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1198
1199         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1200                                           FORCE_EOV_MAX_REZ_CNT(255)));
1201
1202         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1203                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1204
1205         WREG32(VGT_GS_VERTEX_REUSE, 16);
1206         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1207
1208         WREG32(CB_PERF_CTR0_SEL_0, 0);
1209         WREG32(CB_PERF_CTR0_SEL_1, 0);
1210         WREG32(CB_PERF_CTR1_SEL_0, 0);
1211         WREG32(CB_PERF_CTR1_SEL_1, 0);
1212         WREG32(CB_PERF_CTR2_SEL_0, 0);
1213         WREG32(CB_PERF_CTR2_SEL_1, 0);
1214         WREG32(CB_PERF_CTR3_SEL_0, 0);
1215         WREG32(CB_PERF_CTR3_SEL_1, 0);
1216
1217         tmp = RREG32(HDP_MISC_CNTL);
1218         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1219         WREG32(HDP_MISC_CNTL, tmp);
1220
1221         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1222         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1223
1224         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1225
1226         udelay(50);
1227
1228         /* set clockgating golden values on TN */
1229         if (rdev->family == CHIP_ARUBA) {
1230                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1231                 tmp &= ~0x00380000;
1232                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1233                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1234                 tmp &= ~0x0e000000;
1235                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1236         }
1237 }
1238
1239 /*
1240  * GART
1241  */
1242 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1243 {
1244         /* flush hdp cache */
1245         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1246
1247         /* bits 0-7 are the VM contexts0-7 */
1248         WREG32(VM_INVALIDATE_REQUEST, 1);
1249 }
1250
1251 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1252 {
1253         int i, r;
1254
1255         if (rdev->gart.robj == NULL) {
1256                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1257                 return -EINVAL;
1258         }
1259         r = radeon_gart_table_vram_pin(rdev);
1260         if (r)
1261                 return r;
1262         /* Setup TLB control */
1263         WREG32(MC_VM_MX_L1_TLB_CNTL,
1264                (0xA << 7) |
1265                ENABLE_L1_TLB |
1266                ENABLE_L1_FRAGMENT_PROCESSING |
1267                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1268                ENABLE_ADVANCED_DRIVER_MODEL |
1269                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1270         /* Setup L2 cache */
1271         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1272                ENABLE_L2_FRAGMENT_PROCESSING |
1273                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1274                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1275                EFFECTIVE_L2_QUEUE_SIZE(7) |
1276                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1277         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1278         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1279                BANK_SELECT(6) |
1280                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1281         /* setup context0 */
1282         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1283         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1284         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1285         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1286                         (u32)(rdev->dummy_page.addr >> 12));
1287         WREG32(VM_CONTEXT0_CNTL2, 0);
1288         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1289                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1290
1291         WREG32(0x15D4, 0);
1292         WREG32(0x15D8, 0);
1293         WREG32(0x15DC, 0);
1294
1295         /* empty context1-7 */
1296         /* Assign the pt base to something valid for now; the pts used for
1297          * the VMs are determined by the application and setup and assigned
1298          * on the fly in the vm part of radeon_gart.c
1299          */
1300         for (i = 1; i < 8; i++) {
1301                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1302                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1303                         rdev->vm_manager.max_pfn - 1);
1304                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1305                        rdev->vm_manager.saved_table_addr[i]);
1306         }
1307
1308         /* enable context1-7 */
1309         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1310                (u32)(rdev->dummy_page.addr >> 12));
1311         WREG32(VM_CONTEXT1_CNTL2, 4);
1312         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1313                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1314                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1315                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1316                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1317                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1318                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1319                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1320                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1321                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1322                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1323                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1324                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1325                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1326
1327         cayman_pcie_gart_tlb_flush(rdev);
1328         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1329                  (unsigned)(rdev->mc.gtt_size >> 20),
1330                  (unsigned long long)rdev->gart.table_addr);
1331         rdev->gart.ready = true;
1332         return 0;
1333 }
1334
1335 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1336 {
1337         unsigned i;
1338
1339         for (i = 1; i < 8; ++i) {
1340                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1341                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1342         }
1343
1344         /* Disable all tables */
1345         WREG32(VM_CONTEXT0_CNTL, 0);
1346         WREG32(VM_CONTEXT1_CNTL, 0);
1347         /* Setup TLB control */
1348         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1349                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1350                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1351         /* Setup L2 cache */
1352         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1353                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1354                EFFECTIVE_L2_QUEUE_SIZE(7) |
1355                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1356         WREG32(VM_L2_CNTL2, 0);
1357         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1358                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1359         radeon_gart_table_vram_unpin(rdev);
1360 }
1361
1362 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1363 {
1364         cayman_pcie_gart_disable(rdev);
1365         radeon_gart_table_vram_free(rdev);
1366         radeon_gart_fini(rdev);
1367 }
1368
1369 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1370                               int ring, u32 cp_int_cntl)
1371 {
1372         WREG32(SRBM_GFX_CNTL, RINGID(ring));
1373         WREG32(CP_INT_CNTL, cp_int_cntl);
1374 }
1375
1376 /*
1377  * CP.
1378  */
1379 void cayman_fence_ring_emit(struct radeon_device *rdev,
1380                             struct radeon_fence *fence)
1381 {
1382         struct radeon_ring *ring = &rdev->ring[fence->ring];
1383         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1384         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1385                 PACKET3_SH_ACTION_ENA;
1386
1387         /* flush read cache over gart for this vmid */
1388         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1389         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1390         radeon_ring_write(ring, 0xFFFFFFFF);
1391         radeon_ring_write(ring, 0);
1392         radeon_ring_write(ring, 10); /* poll interval */
1393         /* EVENT_WRITE_EOP - flush caches, send int */
1394         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1395         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1396         radeon_ring_write(ring, lower_32_bits(addr));
1397         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1398         radeon_ring_write(ring, fence->seq);
1399         radeon_ring_write(ring, 0);
1400 }
1401
1402 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1403 {
1404         struct radeon_ring *ring = &rdev->ring[ib->ring];
1405         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1406         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1407                 PACKET3_SH_ACTION_ENA;
1408
1409         /* set to DX10/11 mode */
1410         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1411         radeon_ring_write(ring, 1);
1412
1413         if (ring->rptr_save_reg) {
1414                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1415                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1416                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1417                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1418                 radeon_ring_write(ring, next_rptr);
1419         }
1420
1421         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1422         radeon_ring_write(ring,
1423 #ifdef __BIG_ENDIAN
1424                           (2 << 0) |
1425 #endif
1426                           (ib->gpu_addr & 0xFFFFFFFC));
1427         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1428         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1429
1430         /* flush read cache over gart for this vmid */
1431         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1432         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1433         radeon_ring_write(ring, 0xFFFFFFFF);
1434         radeon_ring_write(ring, 0);
1435         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1436 }
1437
1438 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1439 {
1440         if (enable)
1441                 WREG32(CP_ME_CNTL, 0);
1442         else {
1443                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1444                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1445                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1446                 WREG32(SCRATCH_UMSK, 0);
1447                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1448         }
1449 }
1450
1451 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1452                         struct radeon_ring *ring)
1453 {
1454         u32 rptr;
1455
1456         if (rdev->wb.enabled)
1457                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1458         else {
1459                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1460                         rptr = RREG32(CP_RB0_RPTR);
1461                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1462                         rptr = RREG32(CP_RB1_RPTR);
1463                 else
1464                         rptr = RREG32(CP_RB2_RPTR);
1465         }
1466
1467         return rptr;
1468 }
1469
1470 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1471                         struct radeon_ring *ring)
1472 {
1473         u32 wptr;
1474
1475         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1476                 wptr = RREG32(CP_RB0_WPTR);
1477         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1478                 wptr = RREG32(CP_RB1_WPTR);
1479         else
1480                 wptr = RREG32(CP_RB2_WPTR);
1481
1482         return wptr;
1483 }
1484
1485 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1486                          struct radeon_ring *ring)
1487 {
1488         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1489                 WREG32(CP_RB0_WPTR, ring->wptr);
1490                 (void)RREG32(CP_RB0_WPTR);
1491         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1492                 WREG32(CP_RB1_WPTR, ring->wptr);
1493                 (void)RREG32(CP_RB1_WPTR);
1494         } else {
1495                 WREG32(CP_RB2_WPTR, ring->wptr);
1496                 (void)RREG32(CP_RB2_WPTR);
1497         }
1498 }
1499
1500 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1501 {
1502         const __be32 *fw_data;
1503         int i;
1504
1505         if (!rdev->me_fw || !rdev->pfp_fw)
1506                 return -EINVAL;
1507
1508         cayman_cp_enable(rdev, false);
1509
1510         fw_data = (const __be32 *)rdev->pfp_fw->data;
1511         WREG32(CP_PFP_UCODE_ADDR, 0);
1512         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1513                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1514         WREG32(CP_PFP_UCODE_ADDR, 0);
1515
1516         fw_data = (const __be32 *)rdev->me_fw->data;
1517         WREG32(CP_ME_RAM_WADDR, 0);
1518         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1519                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1520
1521         WREG32(CP_PFP_UCODE_ADDR, 0);
1522         WREG32(CP_ME_RAM_WADDR, 0);
1523         WREG32(CP_ME_RAM_RADDR, 0);
1524         return 0;
1525 }
1526
1527 static int cayman_cp_start(struct radeon_device *rdev)
1528 {
1529         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1530         int r, i;
1531
1532         r = radeon_ring_lock(rdev, ring, 7);
1533         if (r) {
1534                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1535                 return r;
1536         }
1537         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1538         radeon_ring_write(ring, 0x1);
1539         radeon_ring_write(ring, 0x0);
1540         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1541         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1542         radeon_ring_write(ring, 0);
1543         radeon_ring_write(ring, 0);
1544         radeon_ring_unlock_commit(rdev, ring, false);
1545
1546         cayman_cp_enable(rdev, true);
1547
1548         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1549         if (r) {
1550                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1551                 return r;
1552         }
1553
1554         /* setup clear context state */
1555         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1556         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1557
1558         for (i = 0; i < cayman_default_size; i++)
1559                 radeon_ring_write(ring, cayman_default_state[i]);
1560
1561         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1562         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1563
1564         /* set clear context state */
1565         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1566         radeon_ring_write(ring, 0);
1567
1568         /* SQ_VTX_BASE_VTX_LOC */
1569         radeon_ring_write(ring, 0xc0026f00);
1570         radeon_ring_write(ring, 0x00000000);
1571         radeon_ring_write(ring, 0x00000000);
1572         radeon_ring_write(ring, 0x00000000);
1573
1574         /* Clear consts */
1575         radeon_ring_write(ring, 0xc0036f00);
1576         radeon_ring_write(ring, 0x00000bc4);
1577         radeon_ring_write(ring, 0xffffffff);
1578         radeon_ring_write(ring, 0xffffffff);
1579         radeon_ring_write(ring, 0xffffffff);
1580
1581         radeon_ring_write(ring, 0xc0026900);
1582         radeon_ring_write(ring, 0x00000316);
1583         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1584         radeon_ring_write(ring, 0x00000010); /*  */
1585
1586         radeon_ring_unlock_commit(rdev, ring, false);
1587
1588         /* XXX init other rings */
1589
1590         return 0;
1591 }
1592
1593 static void cayman_cp_fini(struct radeon_device *rdev)
1594 {
1595         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1596         cayman_cp_enable(rdev, false);
1597         radeon_ring_fini(rdev, ring);
1598         radeon_scratch_free(rdev, ring->rptr_save_reg);
1599 }
1600
1601 static int cayman_cp_resume(struct radeon_device *rdev)
1602 {
1603         static const int ridx[] = {
1604                 RADEON_RING_TYPE_GFX_INDEX,
1605                 CAYMAN_RING_TYPE_CP1_INDEX,
1606                 CAYMAN_RING_TYPE_CP2_INDEX
1607         };
1608         static const unsigned cp_rb_cntl[] = {
1609                 CP_RB0_CNTL,
1610                 CP_RB1_CNTL,
1611                 CP_RB2_CNTL,
1612         };
1613         static const unsigned cp_rb_rptr_addr[] = {
1614                 CP_RB0_RPTR_ADDR,
1615                 CP_RB1_RPTR_ADDR,
1616                 CP_RB2_RPTR_ADDR
1617         };
1618         static const unsigned cp_rb_rptr_addr_hi[] = {
1619                 CP_RB0_RPTR_ADDR_HI,
1620                 CP_RB1_RPTR_ADDR_HI,
1621                 CP_RB2_RPTR_ADDR_HI
1622         };
1623         static const unsigned cp_rb_base[] = {
1624                 CP_RB0_BASE,
1625                 CP_RB1_BASE,
1626                 CP_RB2_BASE
1627         };
1628         static const unsigned cp_rb_rptr[] = {
1629                 CP_RB0_RPTR,
1630                 CP_RB1_RPTR,
1631                 CP_RB2_RPTR
1632         };
1633         static const unsigned cp_rb_wptr[] = {
1634                 CP_RB0_WPTR,
1635                 CP_RB1_WPTR,
1636                 CP_RB2_WPTR
1637         };
1638         struct radeon_ring *ring;
1639         int i, r;
1640
1641         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1642         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1643                                  SOFT_RESET_PA |
1644                                  SOFT_RESET_SH |
1645                                  SOFT_RESET_VGT |
1646                                  SOFT_RESET_SPI |
1647                                  SOFT_RESET_SX));
1648         RREG32(GRBM_SOFT_RESET);
1649         mdelay(15);
1650         WREG32(GRBM_SOFT_RESET, 0);
1651         RREG32(GRBM_SOFT_RESET);
1652
1653         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1654         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1655
1656         /* Set the write pointer delay */
1657         WREG32(CP_RB_WPTR_DELAY, 0);
1658
1659         WREG32(CP_DEBUG, (1 << 27));
1660
1661         /* set the wb address whether it's enabled or not */
1662         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1663         WREG32(SCRATCH_UMSK, 0xff);
1664
1665         for (i = 0; i < 3; ++i) {
1666                 uint32_t rb_cntl;
1667                 uint64_t addr;
1668
1669                 /* Set ring buffer size */
1670                 ring = &rdev->ring[ridx[i]];
1671                 rb_cntl = order_base_2(ring->ring_size / 8);
1672                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1673 #ifdef __BIG_ENDIAN
1674                 rb_cntl |= BUF_SWAP_32BIT;
1675 #endif
1676                 WREG32(cp_rb_cntl[i], rb_cntl);
1677
1678                 /* set the wb address whether it's enabled or not */
1679                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1680                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1681                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1682         }
1683
1684         /* set the rb base addr, this causes an internal reset of ALL rings */
1685         for (i = 0; i < 3; ++i) {
1686                 ring = &rdev->ring[ridx[i]];
1687                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1688         }
1689
1690         for (i = 0; i < 3; ++i) {
1691                 /* Initialize the ring buffer's read and write pointers */
1692                 ring = &rdev->ring[ridx[i]];
1693                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1694
1695                 ring->wptr = 0;
1696                 WREG32(cp_rb_rptr[i], 0);
1697                 WREG32(cp_rb_wptr[i], ring->wptr);
1698
1699                 mdelay(1);
1700                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1701         }
1702
1703         /* start the rings */
1704         cayman_cp_start(rdev);
1705         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1706         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1707         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1708         /* this only test cp0 */
1709         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1710         if (r) {
1711                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1712                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1713                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1714                 return r;
1715         }
1716
1717         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1718                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1719
1720         return 0;
1721 }
1722
1723 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1724 {
1725         u32 reset_mask = 0;
1726         u32 tmp;
1727
1728         /* GRBM_STATUS */
1729         tmp = RREG32(GRBM_STATUS);
1730         if (tmp & (PA_BUSY | SC_BUSY |
1731                    SH_BUSY | SX_BUSY |
1732                    TA_BUSY | VGT_BUSY |
1733                    DB_BUSY | CB_BUSY |
1734                    GDS_BUSY | SPI_BUSY |
1735                    IA_BUSY | IA_BUSY_NO_DMA))
1736                 reset_mask |= RADEON_RESET_GFX;
1737
1738         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1739                    CP_BUSY | CP_COHERENCY_BUSY))
1740                 reset_mask |= RADEON_RESET_CP;
1741
1742         if (tmp & GRBM_EE_BUSY)
1743                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1744
1745         /* DMA_STATUS_REG 0 */
1746         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1747         if (!(tmp & DMA_IDLE))
1748                 reset_mask |= RADEON_RESET_DMA;
1749
1750         /* DMA_STATUS_REG 1 */
1751         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1752         if (!(tmp & DMA_IDLE))
1753                 reset_mask |= RADEON_RESET_DMA1;
1754
1755         /* SRBM_STATUS2 */
1756         tmp = RREG32(SRBM_STATUS2);
1757         if (tmp & DMA_BUSY)
1758                 reset_mask |= RADEON_RESET_DMA;
1759
1760         if (tmp & DMA1_BUSY)
1761                 reset_mask |= RADEON_RESET_DMA1;
1762
1763         /* SRBM_STATUS */
1764         tmp = RREG32(SRBM_STATUS);
1765         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1766                 reset_mask |= RADEON_RESET_RLC;
1767
1768         if (tmp & IH_BUSY)
1769                 reset_mask |= RADEON_RESET_IH;
1770
1771         if (tmp & SEM_BUSY)
1772                 reset_mask |= RADEON_RESET_SEM;
1773
1774         if (tmp & GRBM_RQ_PENDING)
1775                 reset_mask |= RADEON_RESET_GRBM;
1776
1777         if (tmp & VMC_BUSY)
1778                 reset_mask |= RADEON_RESET_VMC;
1779
1780         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1781                    MCC_BUSY | MCD_BUSY))
1782                 reset_mask |= RADEON_RESET_MC;
1783
1784         if (evergreen_is_display_hung(rdev))
1785                 reset_mask |= RADEON_RESET_DISPLAY;
1786
1787         /* VM_L2_STATUS */
1788         tmp = RREG32(VM_L2_STATUS);
1789         if (tmp & L2_BUSY)
1790                 reset_mask |= RADEON_RESET_VMC;
1791
1792         /* Skip MC reset as it's mostly likely not hung, just busy */
1793         if (reset_mask & RADEON_RESET_MC) {
1794                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1795                 reset_mask &= ~RADEON_RESET_MC;
1796         }
1797
1798         return reset_mask;
1799 }
1800
1801 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1802 {
1803         struct evergreen_mc_save save;
1804         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1805         u32 tmp;
1806
1807         if (reset_mask == 0)
1808                 return;
1809
1810         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1811
1812         evergreen_print_gpu_status_regs(rdev);
1813         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1814                  RREG32(0x14F8));
1815         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1816                  RREG32(0x14D8));
1817         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1818                  RREG32(0x14FC));
1819         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1820                  RREG32(0x14DC));
1821
1822         /* Disable CP parsing/prefetching */
1823         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1824
1825         if (reset_mask & RADEON_RESET_DMA) {
1826                 /* dma0 */
1827                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1828                 tmp &= ~DMA_RB_ENABLE;
1829                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1830         }
1831
1832         if (reset_mask & RADEON_RESET_DMA1) {
1833                 /* dma1 */
1834                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1835                 tmp &= ~DMA_RB_ENABLE;
1836                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1837         }
1838
1839         udelay(50);
1840
1841         evergreen_mc_stop(rdev, &save);
1842         if (evergreen_mc_wait_for_idle(rdev)) {
1843                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1844         }
1845
1846         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1847                 grbm_soft_reset = SOFT_RESET_CB |
1848                         SOFT_RESET_DB |
1849                         SOFT_RESET_GDS |
1850                         SOFT_RESET_PA |
1851                         SOFT_RESET_SC |
1852                         SOFT_RESET_SPI |
1853                         SOFT_RESET_SH |
1854                         SOFT_RESET_SX |
1855                         SOFT_RESET_TC |
1856                         SOFT_RESET_TA |
1857                         SOFT_RESET_VGT |
1858                         SOFT_RESET_IA;
1859         }
1860
1861         if (reset_mask & RADEON_RESET_CP) {
1862                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1863
1864                 srbm_soft_reset |= SOFT_RESET_GRBM;
1865         }
1866
1867         if (reset_mask & RADEON_RESET_DMA)
1868                 srbm_soft_reset |= SOFT_RESET_DMA;
1869
1870         if (reset_mask & RADEON_RESET_DMA1)
1871                 srbm_soft_reset |= SOFT_RESET_DMA1;
1872
1873         if (reset_mask & RADEON_RESET_DISPLAY)
1874                 srbm_soft_reset |= SOFT_RESET_DC;
1875
1876         if (reset_mask & RADEON_RESET_RLC)
1877                 srbm_soft_reset |= SOFT_RESET_RLC;
1878
1879         if (reset_mask & RADEON_RESET_SEM)
1880                 srbm_soft_reset |= SOFT_RESET_SEM;
1881
1882         if (reset_mask & RADEON_RESET_IH)
1883                 srbm_soft_reset |= SOFT_RESET_IH;
1884
1885         if (reset_mask & RADEON_RESET_GRBM)
1886                 srbm_soft_reset |= SOFT_RESET_GRBM;
1887
1888         if (reset_mask & RADEON_RESET_VMC)
1889                 srbm_soft_reset |= SOFT_RESET_VMC;
1890
1891         if (!(rdev->flags & RADEON_IS_IGP)) {
1892                 if (reset_mask & RADEON_RESET_MC)
1893                         srbm_soft_reset |= SOFT_RESET_MC;
1894         }
1895
1896         if (grbm_soft_reset) {
1897                 tmp = RREG32(GRBM_SOFT_RESET);
1898                 tmp |= grbm_soft_reset;
1899                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1900                 WREG32(GRBM_SOFT_RESET, tmp);
1901                 tmp = RREG32(GRBM_SOFT_RESET);
1902
1903                 udelay(50);
1904
1905                 tmp &= ~grbm_soft_reset;
1906                 WREG32(GRBM_SOFT_RESET, tmp);
1907                 tmp = RREG32(GRBM_SOFT_RESET);
1908         }
1909
1910         if (srbm_soft_reset) {
1911                 tmp = RREG32(SRBM_SOFT_RESET);
1912                 tmp |= srbm_soft_reset;
1913                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1914                 WREG32(SRBM_SOFT_RESET, tmp);
1915                 tmp = RREG32(SRBM_SOFT_RESET);
1916
1917                 udelay(50);
1918
1919                 tmp &= ~srbm_soft_reset;
1920                 WREG32(SRBM_SOFT_RESET, tmp);
1921                 tmp = RREG32(SRBM_SOFT_RESET);
1922         }
1923
1924         /* Wait a little for things to settle down */
1925         udelay(50);
1926
1927         evergreen_mc_resume(rdev, &save);
1928         udelay(50);
1929
1930         evergreen_print_gpu_status_regs(rdev);
1931 }
1932
1933 int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1934 {
1935         u32 reset_mask;
1936
1937         if (hard) {
1938                 evergreen_gpu_pci_config_reset(rdev);
1939                 return 0;
1940         }
1941
1942         reset_mask = cayman_gpu_check_soft_reset(rdev);
1943
1944         if (reset_mask)
1945                 r600_set_bios_scratch_engine_hung(rdev, true);
1946
1947         cayman_gpu_soft_reset(rdev, reset_mask);
1948
1949         reset_mask = cayman_gpu_check_soft_reset(rdev);
1950
1951         if (reset_mask)
1952                 evergreen_gpu_pci_config_reset(rdev);
1953
1954         r600_set_bios_scratch_engine_hung(rdev, false);
1955
1956         return 0;
1957 }
1958
1959 /**
1960  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1961  *
1962  * @rdev: radeon_device pointer
1963  * @ring: radeon_ring structure holding ring information
1964  *
1965  * Check if the GFX engine is locked up.
1966  * Returns true if the engine appears to be locked up, false if not.
1967  */
1968 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1969 {
1970         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1971
1972         if (!(reset_mask & (RADEON_RESET_GFX |
1973                             RADEON_RESET_COMPUTE |
1974                             RADEON_RESET_CP))) {
1975                 radeon_ring_lockup_update(rdev, ring);
1976                 return false;
1977         }
1978         return radeon_ring_test_lockup(rdev, ring);
1979 }
1980
1981 static void cayman_uvd_init(struct radeon_device *rdev)
1982 {
1983         int r;
1984
1985         if (!rdev->has_uvd)
1986                 return;
1987
1988         r = radeon_uvd_init(rdev);
1989         if (r) {
1990                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1991                 /*
1992                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1993                  * to early fails uvd_v2_2_resume() and thus nothing happens
1994                  * there. So it is pointless to try to go through that code
1995                  * hence why we disable uvd here.
1996                  */
1997                 rdev->has_uvd = 0;
1998                 return;
1999         }
2000         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
2001         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
2002 }
2003
2004 static void cayman_uvd_start(struct radeon_device *rdev)
2005 {
2006         int r;
2007
2008         if (!rdev->has_uvd)
2009                 return;
2010
2011         r = uvd_v2_2_resume(rdev);
2012         if (r) {
2013                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2014                 goto error;
2015         }
2016         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2017         if (r) {
2018                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2019                 goto error;
2020         }
2021         return;
2022
2023 error:
2024         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2025 }
2026
2027 static void cayman_uvd_resume(struct radeon_device *rdev)
2028 {
2029         struct radeon_ring *ring;
2030         int r;
2031
2032         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2033                 return;
2034
2035         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2036         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2037         if (r) {
2038                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2039                 return;
2040         }
2041         r = uvd_v1_0_init(rdev);
2042         if (r) {
2043                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2044                 return;
2045         }
2046 }
2047
2048 static void cayman_vce_init(struct radeon_device *rdev)
2049 {
2050         int r;
2051
2052         /* Only set for CHIP_ARUBA */
2053         if (!rdev->has_vce)
2054                 return;
2055
2056         r = radeon_vce_init(rdev);
2057         if (r) {
2058                 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2059                 /*
2060                  * At this point rdev->vce.vcpu_bo is NULL which trickles down
2061                  * to early fails cayman_vce_start() and thus nothing happens
2062                  * there. So it is pointless to try to go through that code
2063                  * hence why we disable vce here.
2064                  */
2065                 rdev->has_vce = 0;
2066                 return;
2067         }
2068         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2069         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2070         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2071         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2072 }
2073
2074 static void cayman_vce_start(struct radeon_device *rdev)
2075 {
2076         int r;
2077
2078         if (!rdev->has_vce)
2079                 return;
2080
2081         r = radeon_vce_resume(rdev);
2082         if (r) {
2083                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2084                 goto error;
2085         }
2086         r = vce_v1_0_resume(rdev);
2087         if (r) {
2088                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2089                 goto error;
2090         }
2091         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2092         if (r) {
2093                 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2094                 goto error;
2095         }
2096         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2097         if (r) {
2098                 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2099                 goto error;
2100         }
2101         return;
2102
2103 error:
2104         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2105         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2106 }
2107
2108 static void cayman_vce_resume(struct radeon_device *rdev)
2109 {
2110         struct radeon_ring *ring;
2111         int r;
2112
2113         if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2114                 return;
2115
2116         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2117         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2118         if (r) {
2119                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2120                 return;
2121         }
2122         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2123         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2124         if (r) {
2125                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2126                 return;
2127         }
2128         r = vce_v1_0_init(rdev);
2129         if (r) {
2130                 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2131                 return;
2132         }
2133 }
2134
2135 static int cayman_startup(struct radeon_device *rdev)
2136 {
2137         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2138         int r;
2139
2140         /* enable pcie gen2 link */
2141         evergreen_pcie_gen2_enable(rdev);
2142         /* enable aspm */
2143         evergreen_program_aspm(rdev);
2144
2145         /* scratch needs to be initialized before MC */
2146         r = r600_vram_scratch_init(rdev);
2147         if (r)
2148                 return r;
2149
2150         evergreen_mc_program(rdev);
2151
2152         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2153                 r = ni_mc_load_microcode(rdev);
2154                 if (r) {
2155                         DRM_ERROR("Failed to load MC firmware!\n");
2156                         return r;
2157                 }
2158         }
2159
2160         r = cayman_pcie_gart_enable(rdev);
2161         if (r)
2162                 return r;
2163         cayman_gpu_init(rdev);
2164
2165         /* allocate rlc buffers */
2166         if (rdev->flags & RADEON_IS_IGP) {
2167                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2168                 rdev->rlc.reg_list_size =
2169                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2170                 rdev->rlc.cs_data = cayman_cs_data;
2171                 r = sumo_rlc_init(rdev);
2172                 if (r) {
2173                         DRM_ERROR("Failed to init rlc BOs!\n");
2174                         return r;
2175                 }
2176         }
2177
2178         /* allocate wb buffer */
2179         r = radeon_wb_init(rdev);
2180         if (r)
2181                 return r;
2182
2183         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2184         if (r) {
2185                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2186                 return r;
2187         }
2188
2189         cayman_uvd_start(rdev);
2190         cayman_vce_start(rdev);
2191
2192         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2193         if (r) {
2194                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2195                 return r;
2196         }
2197
2198         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2199         if (r) {
2200                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2201                 return r;
2202         }
2203
2204         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2205         if (r) {
2206                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2207                 return r;
2208         }
2209
2210         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2211         if (r) {
2212                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2213                 return r;
2214         }
2215
2216         /* Enable IRQ */
2217         if (!rdev->irq.installed) {
2218                 r = radeon_irq_kms_init(rdev);
2219                 if (r)
2220                         return r;
2221         }
2222
2223         r = r600_irq_init(rdev);
2224         if (r) {
2225                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2226                 radeon_irq_kms_fini(rdev);
2227                 return r;
2228         }
2229         evergreen_irq_set(rdev);
2230
2231         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2232                              RADEON_CP_PACKET2);
2233         if (r)
2234                 return r;
2235
2236         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2237         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2238                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2239         if (r)
2240                 return r;
2241
2242         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2243         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2244                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2245         if (r)
2246                 return r;
2247
2248         r = cayman_cp_load_microcode(rdev);
2249         if (r)
2250                 return r;
2251         r = cayman_cp_resume(rdev);
2252         if (r)
2253                 return r;
2254
2255         r = cayman_dma_resume(rdev);
2256         if (r)
2257                 return r;
2258
2259         cayman_uvd_resume(rdev);
2260         cayman_vce_resume(rdev);
2261
2262         r = radeon_ib_pool_init(rdev);
2263         if (r) {
2264                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2265                 return r;
2266         }
2267
2268         r = radeon_vm_manager_init(rdev);
2269         if (r) {
2270                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2271                 return r;
2272         }
2273
2274         r = radeon_audio_init(rdev);
2275         if (r)
2276                 return r;
2277
2278         return 0;
2279 }
2280
2281 int cayman_resume(struct radeon_device *rdev)
2282 {
2283         int r;
2284
2285         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2286          * posting will perform necessary task to bring back GPU into good
2287          * shape.
2288          */
2289         /* post card */
2290         atom_asic_init(rdev->mode_info.atom_context);
2291
2292         /* init golden registers */
2293         ni_init_golden_registers(rdev);
2294
2295         if (rdev->pm.pm_method == PM_METHOD_DPM)
2296                 radeon_pm_resume(rdev);
2297
2298         rdev->accel_working = true;
2299         r = cayman_startup(rdev);
2300         if (r) {
2301                 DRM_ERROR("cayman startup failed on resume\n");
2302                 rdev->accel_working = false;
2303                 return r;
2304         }
2305         return r;
2306 }
2307
2308 int cayman_suspend(struct radeon_device *rdev)
2309 {
2310         radeon_pm_suspend(rdev);
2311         radeon_audio_fini(rdev);
2312         radeon_vm_manager_fini(rdev);
2313         cayman_cp_enable(rdev, false);
2314         cayman_dma_stop(rdev);
2315         if (rdev->has_uvd) {
2316                 uvd_v1_0_fini(rdev);
2317                 radeon_uvd_suspend(rdev);
2318         }
2319         evergreen_irq_suspend(rdev);
2320         radeon_wb_disable(rdev);
2321         cayman_pcie_gart_disable(rdev);
2322         return 0;
2323 }
2324
2325 /* Plan is to move initialization in that function and use
2326  * helper function so that radeon_device_init pretty much
2327  * do nothing more than calling asic specific function. This
2328  * should also allow to remove a bunch of callback function
2329  * like vram_info.
2330  */
2331 int cayman_init(struct radeon_device *rdev)
2332 {
2333         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2334         int r;
2335
2336         /* Read BIOS */
2337         if (!radeon_get_bios(rdev)) {
2338                 if (ASIC_IS_AVIVO(rdev))
2339                         return -EINVAL;
2340         }
2341         /* Must be an ATOMBIOS */
2342         if (!rdev->is_atom_bios) {
2343                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2344                 return -EINVAL;
2345         }
2346         r = radeon_atombios_init(rdev);
2347         if (r)
2348                 return r;
2349
2350         /* Post card if necessary */
2351         if (!radeon_card_posted(rdev)) {
2352                 if (!rdev->bios) {
2353                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2354                         return -EINVAL;
2355                 }
2356                 DRM_INFO("GPU not posted. posting now...\n");
2357                 atom_asic_init(rdev->mode_info.atom_context);
2358         }
2359         /* init golden registers */
2360         ni_init_golden_registers(rdev);
2361         /* Initialize scratch registers */
2362         r600_scratch_init(rdev);
2363         /* Initialize surface registers */
2364         radeon_surface_init(rdev);
2365         /* Initialize clocks */
2366         radeon_get_clock_info(rdev->ddev);
2367         /* Fence driver */
2368         r = radeon_fence_driver_init(rdev);
2369         if (r)
2370                 return r;
2371         /* initialize memory controller */
2372         r = evergreen_mc_init(rdev);
2373         if (r)
2374                 return r;
2375         /* Memory manager */
2376         r = radeon_bo_init(rdev);
2377         if (r)
2378                 return r;
2379
2380         if (rdev->flags & RADEON_IS_IGP) {
2381                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2382                         r = ni_init_microcode(rdev);
2383                         if (r) {
2384                                 DRM_ERROR("Failed to load firmware!\n");
2385                                 return r;
2386                         }
2387                 }
2388         } else {
2389                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2390                         r = ni_init_microcode(rdev);
2391                         if (r) {
2392                                 DRM_ERROR("Failed to load firmware!\n");
2393                                 return r;
2394                         }
2395                 }
2396         }
2397
2398         /* Initialize power management */
2399         radeon_pm_init(rdev);
2400
2401         ring->ring_obj = NULL;
2402         r600_ring_init(rdev, ring, 1024 * 1024);
2403
2404         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2405         ring->ring_obj = NULL;
2406         r600_ring_init(rdev, ring, 64 * 1024);
2407
2408         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2409         ring->ring_obj = NULL;
2410         r600_ring_init(rdev, ring, 64 * 1024);
2411
2412         cayman_uvd_init(rdev);
2413         cayman_vce_init(rdev);
2414
2415         rdev->ih.ring_obj = NULL;
2416         r600_ih_ring_init(rdev, 64 * 1024);
2417
2418         r = r600_pcie_gart_init(rdev);
2419         if (r)
2420                 return r;
2421
2422         rdev->accel_working = true;
2423         r = cayman_startup(rdev);
2424         if (r) {
2425                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2426                 cayman_cp_fini(rdev);
2427                 cayman_dma_fini(rdev);
2428                 r600_irq_fini(rdev);
2429                 if (rdev->flags & RADEON_IS_IGP)
2430                         sumo_rlc_fini(rdev);
2431                 radeon_wb_fini(rdev);
2432                 radeon_ib_pool_fini(rdev);
2433                 radeon_vm_manager_fini(rdev);
2434                 radeon_irq_kms_fini(rdev);
2435                 cayman_pcie_gart_fini(rdev);
2436                 rdev->accel_working = false;
2437         }
2438
2439         /* Don't start up if the MC ucode is missing.
2440          * The default clocks and voltages before the MC ucode
2441          * is loaded are not suffient for advanced operations.
2442          *
2443          * We can skip this check for TN, because there is no MC
2444          * ucode.
2445          */
2446         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2447                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2448                 return -EINVAL;
2449         }
2450
2451         return 0;
2452 }
2453
2454 void cayman_fini(struct radeon_device *rdev)
2455 {
2456         radeon_pm_fini(rdev);
2457         cayman_cp_fini(rdev);
2458         cayman_dma_fini(rdev);
2459         r600_irq_fini(rdev);
2460         if (rdev->flags & RADEON_IS_IGP)
2461                 sumo_rlc_fini(rdev);
2462         radeon_wb_fini(rdev);
2463         radeon_vm_manager_fini(rdev);
2464         radeon_ib_pool_fini(rdev);
2465         radeon_irq_kms_fini(rdev);
2466         uvd_v1_0_fini(rdev);
2467         radeon_uvd_fini(rdev);
2468         if (rdev->has_vce)
2469                 radeon_vce_fini(rdev);
2470         cayman_pcie_gart_fini(rdev);
2471         r600_vram_scratch_fini(rdev);
2472         radeon_gem_fini(rdev);
2473         radeon_fence_driver_fini(rdev);
2474         radeon_bo_fini(rdev);
2475         radeon_atombios_fini(rdev);
2476         kfree(rdev->bios);
2477         rdev->bios = NULL;
2478 }
2479
2480 /*
2481  * vm
2482  */
2483 int cayman_vm_init(struct radeon_device *rdev)
2484 {
2485         /* number of VMs */
2486         rdev->vm_manager.nvm = 8;
2487         /* base offset of vram pages */
2488         if (rdev->flags & RADEON_IS_IGP) {
2489                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2490                 tmp <<= 22;
2491                 rdev->vm_manager.vram_base_offset = tmp;
2492         } else
2493                 rdev->vm_manager.vram_base_offset = 0;
2494         return 0;
2495 }
2496
2497 void cayman_vm_fini(struct radeon_device *rdev)
2498 {
2499 }
2500
2501 /**
2502  * cayman_vm_decode_fault - print human readable fault info
2503  *
2504  * @rdev: radeon_device pointer
2505  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2506  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2507  *
2508  * Print human readable fault information (cayman/TN).
2509  */
2510 void cayman_vm_decode_fault(struct radeon_device *rdev,
2511                             u32 status, u32 addr)
2512 {
2513         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2514         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2515         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2516         char *block;
2517
2518         switch (mc_id) {
2519         case 32:
2520         case 16:
2521         case 96:
2522         case 80:
2523         case 160:
2524         case 144:
2525         case 224:
2526         case 208:
2527                 block = "CB";
2528                 break;
2529         case 33:
2530         case 17:
2531         case 97:
2532         case 81:
2533         case 161:
2534         case 145:
2535         case 225:
2536         case 209:
2537                 block = "CB_FMASK";
2538                 break;
2539         case 34:
2540         case 18:
2541         case 98:
2542         case 82:
2543         case 162:
2544         case 146:
2545         case 226:
2546         case 210:
2547                 block = "CB_CMASK";
2548                 break;
2549         case 35:
2550         case 19:
2551         case 99:
2552         case 83:
2553         case 163:
2554         case 147:
2555         case 227:
2556         case 211:
2557                 block = "CB_IMMED";
2558                 break;
2559         case 36:
2560         case 20:
2561         case 100:
2562         case 84:
2563         case 164:
2564         case 148:
2565         case 228:
2566         case 212:
2567                 block = "DB";
2568                 break;
2569         case 37:
2570         case 21:
2571         case 101:
2572         case 85:
2573         case 165:
2574         case 149:
2575         case 229:
2576         case 213:
2577                 block = "DB_HTILE";
2578                 break;
2579         case 38:
2580         case 22:
2581         case 102:
2582         case 86:
2583         case 166:
2584         case 150:
2585         case 230:
2586         case 214:
2587                 block = "SX";
2588                 break;
2589         case 39:
2590         case 23:
2591         case 103:
2592         case 87:
2593         case 167:
2594         case 151:
2595         case 231:
2596         case 215:
2597                 block = "DB_STEN";
2598                 break;
2599         case 40:
2600         case 24:
2601         case 104:
2602         case 88:
2603         case 232:
2604         case 216:
2605         case 168:
2606         case 152:
2607                 block = "TC_TFETCH";
2608                 break;
2609         case 41:
2610         case 25:
2611         case 105:
2612         case 89:
2613         case 233:
2614         case 217:
2615         case 169:
2616         case 153:
2617                 block = "TC_VFETCH";
2618                 break;
2619         case 42:
2620         case 26:
2621         case 106:
2622         case 90:
2623         case 234:
2624         case 218:
2625         case 170:
2626         case 154:
2627                 block = "VC";
2628                 break;
2629         case 112:
2630                 block = "CP";
2631                 break;
2632         case 113:
2633         case 114:
2634                 block = "SH";
2635                 break;
2636         case 115:
2637                 block = "VGT";
2638                 break;
2639         case 178:
2640                 block = "IH";
2641                 break;
2642         case 51:
2643                 block = "RLC";
2644                 break;
2645         case 55:
2646                 block = "DMA";
2647                 break;
2648         case 56:
2649                 block = "HDP";
2650                 break;
2651         default:
2652                 block = "unknown";
2653                 break;
2654         }
2655
2656         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2657                protections, vmid, addr,
2658                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2659                block, mc_id);
2660 }
2661
2662 /**
2663  * cayman_vm_flush - vm flush using the CP
2664  *
2665  * @rdev: radeon_device pointer
2666  *
2667  * Update the page table base and flush the VM TLB
2668  * using the CP (cayman-si).
2669  */
2670 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2671                      unsigned vm_id, uint64_t pd_addr)
2672 {
2673         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2674         radeon_ring_write(ring, pd_addr >> 12);
2675
2676         /* flush hdp cache */
2677         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2678         radeon_ring_write(ring, 0x1);
2679
2680         /* bits 0-7 are the VM contexts0-7 */
2681         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2682         radeon_ring_write(ring, 1 << vm_id);
2683
2684         /* wait for the invalidate to complete */
2685         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2686         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2687                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2688         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2689         radeon_ring_write(ring, 0);
2690         radeon_ring_write(ring, 0); /* ref */
2691         radeon_ring_write(ring, 0); /* mask */
2692         radeon_ring_write(ring, 0x20); /* poll interval */
2693
2694         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2695         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2696         radeon_ring_write(ring, 0x0);
2697 }
2698
2699 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2700 {
2701         struct atom_clock_dividers dividers;
2702         int r, i;
2703
2704         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2705                                            ecclk, false, &dividers);
2706         if (r)
2707                 return r;
2708
2709         for (i = 0; i < 100; i++) {
2710                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2711                         break;
2712                 mdelay(10);
2713         }
2714         if (i == 100)
2715                 return -ETIMEDOUT;
2716
2717         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2718
2719         for (i = 0; i < 100; i++) {
2720                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2721                         break;
2722                 mdelay(10);
2723         }
2724         if (i == 100)
2725                 return -ETIMEDOUT;
2726
2727         return 0;
2728 }