drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 /*
  40  * Indirect registers accessor
  41  */
  42 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  43 {
  44         unsigned long flags;
  45         u32 r;
  46
  47         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  48         WREG32(TN_SMC_IND_INDEX_0, (reg));
  49         r = RREG32(TN_SMC_IND_DATA_0);
  50         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  51         return r;
  52 }
  53
  54 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  55 {
  56         unsigned long flags;
  57
  58         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  59         WREG32(TN_SMC_IND_INDEX_0, (reg));
  60         WREG32(TN_SMC_IND_DATA_0, (v));
  61         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  62 }
  63
  64 static const u32 tn_rlc_save_restore_register_list[] =
  65 {
  66         0x98fc,
  67         0x98f0,
  68         0x9834,
  69         0x9838,
  70         0x9870,
  71         0x9874,
  72         0x8a14,
  73         0x8b24,
  74         0x8bcc,
  75         0x8b10,
  76         0x8c30,
  77         0x8d00,
  78         0x8d04,
  79         0x8c00,
  80         0x8c04,
  81         0x8c10,
  82         0x8c14,
  83         0x8d8c,
  84         0x8cf0,
  85         0x8e38,
  86         0x9508,
  87         0x9688,
  88         0x9608,
  89         0x960c,
  90         0x9610,
  91         0x9614,
  92         0x88c4,
  93         0x8978,
  94         0x88d4,
  95         0x900c,
  96         0x9100,
  97         0x913c,
  98         0x90e8,
  99         0x9354,
 100         0xa008,
 101         0x98f8,
 102         0x9148,
 103         0x914c,
 104         0x3f94,
 105         0x98f4,
 106         0x9b7c,
 107         0x3f8c,
 108         0x8950,
 109         0x8954,
 110         0x8a18,
 111         0x8b28,
 112         0x9144,
 113         0x3f90,
 114         0x915c,
 115         0x9160,
 116         0x9178,
 117         0x917c,
 118         0x9180,
 119         0x918c,
 120         0x9190,
 121         0x9194,
 122         0x9198,
 123         0x919c,
 124         0x91a8,
 125         0x91ac,
 126         0x91b0,
 127         0x91b4,
 128         0x91b8,
 129         0x91c4,
 130         0x91c8,
 131         0x91cc,
 132         0x91d0,
 133         0x91d4,
 134         0x91e0,
 135         0x91e4,
 136         0x91ec,
 137         0x91f0,
 138         0x91f4,
 139         0x9200,
 140         0x9204,
 141         0x929c,
 142         0x8030,
 143         0x9150,
 144         0x9a60,
 145         0x920c,
 146         0x9210,
 147         0x9228,
 148         0x922c,
 149         0x9244,
 150         0x9248,
 151         0x91e8,
 152         0x9294,
 153         0x9208,
 154         0x9224,
 155         0x9240,
 156         0x9220,
 157         0x923c,
 158         0x9258,
 159         0x9744,
 160         0xa200,
 161         0xa204,
 162         0xa208,
 163         0xa20c,
 164         0x8d58,
 165         0x9030,
 166         0x9034,
 167         0x9038,
 168         0x903c,
 169         0x9040,
 170         0x9654,
 171         0x897c,
 172         0xa210,
 173         0xa214,
 174         0x9868,
 175         0xa02c,
 176         0x9664,
 177         0x9698,
 178         0x949c,
 179         0x8e10,
 180         0x8e18,
 181         0x8c50,
 182         0x8c58,
 183         0x8c60,
 184         0x8c68,
 185         0x89b4,
 186         0x9830,
 187         0x802c,
 188 };
 189
 190 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 191 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 192 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 193 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 194 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 195 extern void evergreen_mc_program(struct radeon_device *rdev);
 196 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 197 extern int evergreen_mc_init(struct radeon_device *rdev);
 198 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 199 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 200 extern void evergreen_program_aspm(struct radeon_device *rdev);
 201 extern void sumo_rlc_fini(struct radeon_device *rdev);
 202 extern int sumo_rlc_init(struct radeon_device *rdev);
 203 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 204
 205 /* Firmware Names */
 206 /*(DEBLOBBED)*/
 207
 208
 209 static const u32 cayman_golden_registers2[] =
 210 {
 211         0x3e5c, 0xffffffff, 0x00000000,
 212         0x3e48, 0xffffffff, 0x00000000,
 213         0x3e4c, 0xffffffff, 0x00000000,
 214         0x3e64, 0xffffffff, 0x00000000,
 215         0x3e50, 0xffffffff, 0x00000000,
 216         0x3e60, 0xffffffff, 0x00000000
 217 };
 218
 219 static const u32 cayman_golden_registers[] =
 220 {
 221         0x5eb4, 0xffffffff, 0x00000002,
 222         0x5e78, 0x8f311ff1, 0x001000f0,
 223         0x3f90, 0xffff0000, 0xff000000,
 224         0x9148, 0xffff0000, 0xff000000,
 225         0x3f94, 0xffff0000, 0xff000000,
 226         0x914c, 0xffff0000, 0xff000000,
 227         0xc78, 0x00000080, 0x00000080,
 228         0xbd4, 0x70073777, 0x00011003,
 229         0xd02c, 0xbfffff1f, 0x08421000,
 230         0xd0b8, 0x73773777, 0x02011003,
 231         0x5bc0, 0x00200000, 0x50100000,
 232         0x98f8, 0x33773777, 0x02011003,
 233         0x98fc, 0xffffffff, 0x76541032,
 234         0x7030, 0x31000311, 0x00000011,
 235         0x2f48, 0x33773777, 0x42010001,
 236         0x6b28, 0x00000010, 0x00000012,
 237         0x7728, 0x00000010, 0x00000012,
 238         0x10328, 0x00000010, 0x00000012,
 239         0x10f28, 0x00000010, 0x00000012,
 240         0x11b28, 0x00000010, 0x00000012,
 241         0x12728, 0x00000010, 0x00000012,
 242         0x240c, 0x000007ff, 0x00000000,
 243         0x8a14, 0xf000001f, 0x00000007,
 244         0x8b24, 0x3fff3fff, 0x00ff0fff,
 245         0x8b10, 0x0000ff0f, 0x00000000,
 246         0x28a4c, 0x07ffffff, 0x06000000,
 247         0x10c, 0x00000001, 0x00010003,
 248         0xa02c, 0xffffffff, 0x0000009b,
 249         0x913c, 0x0000010f, 0x01000100,
 250         0x8c04, 0xf8ff00ff, 0x40600060,
 251         0x28350, 0x00000f01, 0x00000000,
 252         0x9508, 0x3700001f, 0x00000002,
 253         0x960c, 0xffffffff, 0x54763210,
 254         0x88c4, 0x001f3ae3, 0x00000082,
 255         0x88d0, 0xffffffff, 0x0f40df40,
 256         0x88d4, 0x0000001f, 0x00000010,
 257         0x8974, 0xffffffff, 0x00000000
 258 };
 259
 260 static const u32 dvst_golden_registers2[] =
 261 {
 262         0x8f8, 0xffffffff, 0,
 263         0x8fc, 0x00380000, 0,
 264         0x8f8, 0xffffffff, 1,
 265         0x8fc, 0x0e000000, 0
 266 };
 267
 268 static const u32 dvst_golden_registers[] =
 269 {
 270         0x690, 0x3fff3fff, 0x20c00033,
 271         0x918c, 0x0fff0fff, 0x00010006,
 272         0x91a8, 0x0fff0fff, 0x00010006,
 273         0x9150, 0xffffdfff, 0x6e944040,
 274         0x917c, 0x0fff0fff, 0x00030002,
 275         0x9198, 0x0fff0fff, 0x00030002,
 276         0x915c, 0x0fff0fff, 0x00010000,
 277         0x3f90, 0xffff0001, 0xff000000,
 278         0x9178, 0x0fff0fff, 0x00070000,
 279         0x9194, 0x0fff0fff, 0x00070000,
 280         0x9148, 0xffff0001, 0xff000000,
 281         0x9190, 0x0fff0fff, 0x00090008,
 282         0x91ac, 0x0fff0fff, 0x00090008,
 283         0x3f94, 0xffff0000, 0xff000000,
 284         0x914c, 0xffff0000, 0xff000000,
 285         0x929c, 0x00000fff, 0x00000001,
 286         0x55e4, 0xff607fff, 0xfc000100,
 287         0x8a18, 0xff000fff, 0x00000100,
 288         0x8b28, 0xff000fff, 0x00000100,
 289         0x9144, 0xfffc0fff, 0x00000100,
 290         0x6ed8, 0x00010101, 0x00010000,
 291         0x9830, 0xffffffff, 0x00000000,
 292         0x9834, 0xf00fffff, 0x00000400,
 293         0x9838, 0xfffffffe, 0x00000000,
 294         0xd0c0, 0xff000fff, 0x00000100,
 295         0xd02c, 0xbfffff1f, 0x08421000,
 296         0xd0b8, 0x73773777, 0x12010001,
 297         0x5bb0, 0x000000f0, 0x00000070,
 298         0x98f8, 0x73773777, 0x12010001,
 299         0x98fc, 0xffffffff, 0x00000010,
 300         0x9b7c, 0x00ff0000, 0x00fc0000,
 301         0x8030, 0x00001f0f, 0x0000100a,
 302         0x2f48, 0x73773777, 0x12010001,
 303         0x2408, 0x00030000, 0x000c007f,
 304         0x8a14, 0xf000003f, 0x00000007,
 305         0x8b24, 0x3fff3fff, 0x00ff0fff,
 306         0x8b10, 0x0000ff0f, 0x00000000,
 307         0x28a4c, 0x07ffffff, 0x06000000,
 308         0x4d8, 0x00000fff, 0x00000100,
 309         0xa008, 0xffffffff, 0x00010000,
 310         0x913c, 0xffff03ff, 0x01000100,
 311         0x8c00, 0x000000ff, 0x00000003,
 312         0x8c04, 0xf8ff00ff, 0x40600060,
 313         0x8cf0, 0x1fff1fff, 0x08e00410,
 314         0x28350, 0x00000f01, 0x00000000,
 315         0x9508, 0xf700071f, 0x00000002,
 316         0x960c, 0xffffffff, 0x54763210,
 317         0x20ef8, 0x01ff01ff, 0x00000002,
 318         0x20e98, 0xfffffbff, 0x00200000,
 319         0x2015c, 0xffffffff, 0x00000f40,
 320         0x88c4, 0x001f3ae3, 0x00000082,
 321         0x8978, 0x3fffffff, 0x04050140,
 322         0x88d4, 0x0000001f, 0x00000010,
 323         0x8974, 0xffffffff, 0x00000000
 324 };
 325
 326 static const u32 scrapper_golden_registers[] =
 327 {
 328         0x690, 0x3fff3fff, 0x20c00033,
 329         0x918c, 0x0fff0fff, 0x00010006,
 330         0x918c, 0x0fff0fff, 0x00010006,
 331         0x91a8, 0x0fff0fff, 0x00010006,
 332         0x91a8, 0x0fff0fff, 0x00010006,
 333         0x9150, 0xffffdfff, 0x6e944040,
 334         0x9150, 0xffffdfff, 0x6e944040,
 335         0x917c, 0x0fff0fff, 0x00030002,
 336         0x917c, 0x0fff0fff, 0x00030002,
 337         0x9198, 0x0fff0fff, 0x00030002,
 338         0x9198, 0x0fff0fff, 0x00030002,
 339         0x915c, 0x0fff0fff, 0x00010000,
 340         0x915c, 0x0fff0fff, 0x00010000,
 341         0x3f90, 0xffff0001, 0xff000000,
 342         0x3f90, 0xffff0001, 0xff000000,
 343         0x9178, 0x0fff0fff, 0x00070000,
 344         0x9178, 0x0fff0fff, 0x00070000,
 345         0x9194, 0x0fff0fff, 0x00070000,
 346         0x9194, 0x0fff0fff, 0x00070000,
 347         0x9148, 0xffff0001, 0xff000000,
 348         0x9148, 0xffff0001, 0xff000000,
 349         0x9190, 0x0fff0fff, 0x00090008,
 350         0x9190, 0x0fff0fff, 0x00090008,
 351         0x91ac, 0x0fff0fff, 0x00090008,
 352         0x91ac, 0x0fff0fff, 0x00090008,
 353         0x3f94, 0xffff0000, 0xff000000,
 354         0x3f94, 0xffff0000, 0xff000000,
 355         0x914c, 0xffff0000, 0xff000000,
 356         0x914c, 0xffff0000, 0xff000000,
 357         0x929c, 0x00000fff, 0x00000001,
 358         0x929c, 0x00000fff, 0x00000001,
 359         0x55e4, 0xff607fff, 0xfc000100,
 360         0x8a18, 0xff000fff, 0x00000100,
 361         0x8a18, 0xff000fff, 0x00000100,
 362         0x8b28, 0xff000fff, 0x00000100,
 363         0x8b28, 0xff000fff, 0x00000100,
 364         0x9144, 0xfffc0fff, 0x00000100,
 365         0x9144, 0xfffc0fff, 0x00000100,
 366         0x6ed8, 0x00010101, 0x00010000,
 367         0x9830, 0xffffffff, 0x00000000,
 368         0x9830, 0xffffffff, 0x00000000,
 369         0x9834, 0xf00fffff, 0x00000400,
 370         0x9834, 0xf00fffff, 0x00000400,
 371         0x9838, 0xfffffffe, 0x00000000,
 372         0x9838, 0xfffffffe, 0x00000000,
 373         0xd0c0, 0xff000fff, 0x00000100,
 374         0xd02c, 0xbfffff1f, 0x08421000,
 375         0xd02c, 0xbfffff1f, 0x08421000,
 376         0xd0b8, 0x73773777, 0x12010001,
 377         0xd0b8, 0x73773777, 0x12010001,
 378         0x5bb0, 0x000000f0, 0x00000070,
 379         0x98f8, 0x73773777, 0x12010001,
 380         0x98f8, 0x73773777, 0x12010001,
 381         0x98fc, 0xffffffff, 0x00000010,
 382         0x98fc, 0xffffffff, 0x00000010,
 383         0x9b7c, 0x00ff0000, 0x00fc0000,
 384         0x9b7c, 0x00ff0000, 0x00fc0000,
 385         0x8030, 0x00001f0f, 0x0000100a,
 386         0x8030, 0x00001f0f, 0x0000100a,
 387         0x2f48, 0x73773777, 0x12010001,
 388         0x2f48, 0x73773777, 0x12010001,
 389         0x2408, 0x00030000, 0x000c007f,
 390         0x8a14, 0xf000003f, 0x00000007,
 391         0x8a14, 0xf000003f, 0x00000007,
 392         0x8b24, 0x3fff3fff, 0x00ff0fff,
 393         0x8b24, 0x3fff3fff, 0x00ff0fff,
 394         0x8b10, 0x0000ff0f, 0x00000000,
 395         0x8b10, 0x0000ff0f, 0x00000000,
 396         0x28a4c, 0x07ffffff, 0x06000000,
 397         0x28a4c, 0x07ffffff, 0x06000000,
 398         0x4d8, 0x00000fff, 0x00000100,
 399         0x4d8, 0x00000fff, 0x00000100,
 400         0xa008, 0xffffffff, 0x00010000,
 401         0xa008, 0xffffffff, 0x00010000,
 402         0x913c, 0xffff03ff, 0x01000100,
 403         0x913c, 0xffff03ff, 0x01000100,
 404         0x90e8, 0x001fffff, 0x010400c0,
 405         0x8c00, 0x000000ff, 0x00000003,
 406         0x8c00, 0x000000ff, 0x00000003,
 407         0x8c04, 0xf8ff00ff, 0x40600060,
 408         0x8c04, 0xf8ff00ff, 0x40600060,
 409         0x8c30, 0x0000000f, 0x00040005,
 410         0x8cf0, 0x1fff1fff, 0x08e00410,
 411         0x8cf0, 0x1fff1fff, 0x08e00410,
 412         0x900c, 0x00ffffff, 0x0017071f,
 413         0x28350, 0x00000f01, 0x00000000,
 414         0x28350, 0x00000f01, 0x00000000,
 415         0x9508, 0xf700071f, 0x00000002,
 416         0x9508, 0xf700071f, 0x00000002,
 417         0x9688, 0x00300000, 0x0017000f,
 418         0x960c, 0xffffffff, 0x54763210,
 419         0x960c, 0xffffffff, 0x54763210,
 420         0x20ef8, 0x01ff01ff, 0x00000002,
 421         0x20e98, 0xfffffbff, 0x00200000,
 422         0x2015c, 0xffffffff, 0x00000f40,
 423         0x88c4, 0x001f3ae3, 0x00000082,
 424         0x88c4, 0x001f3ae3, 0x00000082,
 425         0x8978, 0x3fffffff, 0x04050140,
 426         0x8978, 0x3fffffff, 0x04050140,
 427         0x88d4, 0x0000001f, 0x00000010,
 428         0x88d4, 0x0000001f, 0x00000010,
 429         0x8974, 0xffffffff, 0x00000000,
 430         0x8974, 0xffffffff, 0x00000000
 431 };
 432
 433 static void ni_init_golden_registers(struct radeon_device *rdev)
 434 {
 435         switch (rdev->family) {
 436         case CHIP_CAYMAN:
 437                 radeon_program_register_sequence(rdev,
 438                                                  cayman_golden_registers,
 439                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 440                 radeon_program_register_sequence(rdev,
 441                                                  cayman_golden_registers2,
 442                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 443                 break;
 444         case CHIP_ARUBA:
 445                 if ((rdev->pdev->device == 0x9900) ||
 446                     (rdev->pdev->device == 0x9901) ||
 447                     (rdev->pdev->device == 0x9903) ||
 448                     (rdev->pdev->device == 0x9904) ||
 449                     (rdev->pdev->device == 0x9905) ||
 450                     (rdev->pdev->device == 0x9906) ||
 451                     (rdev->pdev->device == 0x9907) ||
 452                     (rdev->pdev->device == 0x9908) ||
 453                     (rdev->pdev->device == 0x9909) ||
 454                     (rdev->pdev->device == 0x990A) ||
 455                     (rdev->pdev->device == 0x990B) ||
 456                     (rdev->pdev->device == 0x990C) ||
 457                     (rdev->pdev->device == 0x990D) ||
 458                     (rdev->pdev->device == 0x990E) ||
 459                     (rdev->pdev->device == 0x990F) ||
 460                     (rdev->pdev->device == 0x9910) ||
 461                     (rdev->pdev->device == 0x9913) ||
 462                     (rdev->pdev->device == 0x9917) ||
 463                     (rdev->pdev->device == 0x9918)) {
 464                         radeon_program_register_sequence(rdev,
 465                                                          dvst_golden_registers,
 466                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 467                         radeon_program_register_sequence(rdev,
 468                                                          dvst_golden_registers2,
 469                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 470                 } else {
 471                         radeon_program_register_sequence(rdev,
 472                                                          scrapper_golden_registers,
 473                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 474                         radeon_program_register_sequence(rdev,
 475                                                          dvst_golden_registers2,
 476                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 477                 }
 478                 break;
 479         default:
 480                 break;
 481         }
 482 }
 483
 484 #define BTC_IO_MC_REGS_SIZE 29
 485
 486 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 487         {0x00000077, 0xff010100},
 488         {0x00000078, 0x00000000},
 489         {0x00000079, 0x00001434},
 490         {0x0000007a, 0xcc08ec08},
 491         {0x0000007b, 0x00040000},
 492         {0x0000007c, 0x000080c0},
 493         {0x0000007d, 0x09000000},
 494         {0x0000007e, 0x00210404},
 495         {0x00000081, 0x08a8e800},
 496         {0x00000082, 0x00030444},
 497         {0x00000083, 0x00000000},
 498         {0x00000085, 0x00000001},
 499         {0x00000086, 0x00000002},
 500         {0x00000087, 0x48490000},
 501         {0x00000088, 0x20244647},
 502         {0x00000089, 0x00000005},
 503         {0x0000008b, 0x66030000},
 504         {0x0000008c, 0x00006603},
 505         {0x0000008d, 0x00000100},
 506         {0x0000008f, 0x00001c0a},
 507         {0x00000090, 0xff000001},
 508         {0x00000094, 0x00101101},
 509         {0x00000095, 0x00000fff},
 510         {0x00000096, 0x00116fff},
 511         {0x00000097, 0x60010000},
 512         {0x00000098, 0x10010000},
 513         {0x00000099, 0x00006000},
 514         {0x0000009a, 0x00001000},
 515         {0x0000009f, 0x00946a00}
 516 };
 517
 518 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 519         {0x00000077, 0xff010100},
 520         {0x00000078, 0x00000000},
 521         {0x00000079, 0x00001434},
 522         {0x0000007a, 0xcc08ec08},
 523         {0x0000007b, 0x00040000},
 524         {0x0000007c, 0x000080c0},
 525         {0x0000007d, 0x09000000},
 526         {0x0000007e, 0x00210404},
 527         {0x00000081, 0x08a8e800},
 528         {0x00000082, 0x00030444},
 529         {0x00000083, 0x00000000},
 530         {0x00000085, 0x00000001},
 531         {0x00000086, 0x00000002},
 532         {0x00000087, 0x48490000},
 533         {0x00000088, 0x20244647},
 534         {0x00000089, 0x00000005},
 535         {0x0000008b, 0x66030000},
 536         {0x0000008c, 0x00006603},
 537         {0x0000008d, 0x00000100},
 538         {0x0000008f, 0x00001c0a},
 539         {0x00000090, 0xff000001},
 540         {0x00000094, 0x00101101},
 541         {0x00000095, 0x00000fff},
 542         {0x00000096, 0x00116fff},
 543         {0x00000097, 0x60010000},
 544         {0x00000098, 0x10010000},
 545         {0x00000099, 0x00006000},
 546         {0x0000009a, 0x00001000},
 547         {0x0000009f, 0x00936a00}
 548 };
 549
 550 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 551         {0x00000077, 0xff010100},
 552         {0x00000078, 0x00000000},
 553         {0x00000079, 0x00001434},
 554         {0x0000007a, 0xcc08ec08},
 555         {0x0000007b, 0x00040000},
 556         {0x0000007c, 0x000080c0},
 557         {0x0000007d, 0x09000000},
 558         {0x0000007e, 0x00210404},
 559         {0x00000081, 0x08a8e800},
 560         {0x00000082, 0x00030444},
 561         {0x00000083, 0x00000000},
 562         {0x00000085, 0x00000001},
 563         {0x00000086, 0x00000002},
 564         {0x00000087, 0x48490000},
 565         {0x00000088, 0x20244647},
 566         {0x00000089, 0x00000005},
 567         {0x0000008b, 0x66030000},
 568         {0x0000008c, 0x00006603},
 569         {0x0000008d, 0x00000100},
 570         {0x0000008f, 0x00001c0a},
 571         {0x00000090, 0xff000001},
 572         {0x00000094, 0x00101101},
 573         {0x00000095, 0x00000fff},
 574         {0x00000096, 0x00116fff},
 575         {0x00000097, 0x60010000},
 576         {0x00000098, 0x10010000},
 577         {0x00000099, 0x00006000},
 578         {0x0000009a, 0x00001000},
 579         {0x0000009f, 0x00916a00}
 580 };
 581
 582 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 583         {0x00000077, 0xff010100},
 584         {0x00000078, 0x00000000},
 585         {0x00000079, 0x00001434},
 586         {0x0000007a, 0xcc08ec08},
 587         {0x0000007b, 0x00040000},
 588         {0x0000007c, 0x000080c0},
 589         {0x0000007d, 0x09000000},
 590         {0x0000007e, 0x00210404},
 591         {0x00000081, 0x08a8e800},
 592         {0x00000082, 0x00030444},
 593         {0x00000083, 0x00000000},
 594         {0x00000085, 0x00000001},
 595         {0x00000086, 0x00000002},
 596         {0x00000087, 0x48490000},
 597         {0x00000088, 0x20244647},
 598         {0x00000089, 0x00000005},
 599         {0x0000008b, 0x66030000},
 600         {0x0000008c, 0x00006603},
 601         {0x0000008d, 0x00000100},
 602         {0x0000008f, 0x00001c0a},
 603         {0x00000090, 0xff000001},
 604         {0x00000094, 0x00101101},
 605         {0x00000095, 0x00000fff},
 606         {0x00000096, 0x00116fff},
 607         {0x00000097, 0x60010000},
 608         {0x00000098, 0x10010000},
 609         {0x00000099, 0x00006000},
 610         {0x0000009a, 0x00001000},
 611         {0x0000009f, 0x00976b00}
 612 };
 613
 614 int ni_mc_load_microcode(struct radeon_device *rdev)
 615 {
 616         const __be32 *fw_data;
 617         u32 mem_type, running, blackout = 0;
 618         u32 *io_mc_regs;
 619         int i, ucode_size, regs_size;
 620
 621         if (!rdev->mc_fw)
 622                 return -EINVAL;
 623
 624         switch (rdev->family) {
 625         case CHIP_BARTS:
 626                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 627                 ucode_size = BTC_MC_UCODE_SIZE;
 628                 regs_size = BTC_IO_MC_REGS_SIZE;
 629                 break;
 630         case CHIP_TURKS:
 631                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 632                 ucode_size = BTC_MC_UCODE_SIZE;
 633                 regs_size = BTC_IO_MC_REGS_SIZE;
 634                 break;
 635         case CHIP_CAICOS:
 636         default:
 637                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 638                 ucode_size = BTC_MC_UCODE_SIZE;
 639                 regs_size = BTC_IO_MC_REGS_SIZE;
 640                 break;
 641         case CHIP_CAYMAN:
 642                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 643                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 644                 regs_size = BTC_IO_MC_REGS_SIZE;
 645                 break;
 646         }
 647
 648         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 649         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 650
 651         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 652                 if (running) {
 653                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 654                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 655                 }
 656
 657                 /* reset the engine and set to writable */
 658                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 659                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 660
 661                 /* load mc io regs */
 662                 for (i = 0; i < regs_size; i++) {
 663                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 664                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 665                 }
 666                 /* load the MC ucode */
 667                 fw_data = (const __be32 *)rdev->mc_fw->data;
 668                 for (i = 0; i < ucode_size; i++)
 669                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 670
 671                 /* put the engine back into the active state */
 672                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 673                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 674                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 675
 676                 /* wait for training to complete */
 677                 for (i = 0; i < rdev->usec_timeout; i++) {
 678                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 679                                 break;
 680                         udelay(1);
 681                 }
 682
 683                 if (running)
 684                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 685         }
 686
 687         return 0;
 688 }
 689
 690 int ni_init_microcode(struct radeon_device *rdev)
 691 {
 692         const char *chip_name;
 693         const char *rlc_chip_name;
 694         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 695         size_t smc_req_size = 0;
 696         char fw_name[30];
 697         int err;
 698
 699         DRM_DEBUG("\n");
 700
 701         switch (rdev->family) {
 702         case CHIP_BARTS:
 703                 chip_name = "BARTS";
 704                 rlc_chip_name = "BTC";
 705                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 706                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 707                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 708                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 709                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 710                 break;
 711         case CHIP_TURKS:
 712                 chip_name = "TURKS";
 713                 rlc_chip_name = "BTC";
 714                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 715                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 716                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 717                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 718                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 719                 break;
 720         case CHIP_CAICOS:
 721                 chip_name = "CAICOS";
 722                 rlc_chip_name = "BTC";
 723                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 724                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 725                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 726                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 727                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 728                 break;
 729         case CHIP_CAYMAN:
 730                 chip_name = "CAYMAN";
 731                 rlc_chip_name = "CAYMAN";
 732                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 733                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 734                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 735                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 736                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 737                 break;
 738         case CHIP_ARUBA:
 739                 chip_name = "ARUBA";
 740                 rlc_chip_name = "ARUBA";
 741                 /* pfp/me same size as CAYMAN */
 742                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 743                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 744                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 745                 mc_req_size = 0;
 746                 break;
 747         default: BUG();
 748         }
 749
 750         DRM_INFO("Loading %s Microcode\n", chip_name);
 751
 752         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 753         err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 754         if (err)
 755                 goto out;
 756         if (rdev->pfp_fw->size != pfp_req_size) {
 757                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 758                        rdev->pfp_fw->size, fw_name);
 759                 err = -EINVAL;
 760                 goto out;
 761         }
 762
 763         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 764         err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
 765         if (err)
 766                 goto out;
 767         if (rdev->me_fw->size != me_req_size) {
 768                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 769                        rdev->me_fw->size, fw_name);
 770                 err = -EINVAL;
 771         }
 772
 773         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", rlc_chip_name);
 774         err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 775         if (err)
 776                 goto out;
 777         if (rdev->rlc_fw->size != rlc_req_size) {
 778                 pr_err("ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 779                        rdev->rlc_fw->size, fw_name);
 780                 err = -EINVAL;
 781         }
 782
 783         /* no MC ucode on TN */
 784         if (!(rdev->flags & RADEON_IS_IGP)) {
 785                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 786                 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 787                 if (err)
 788                         goto out;
 789                 if (rdev->mc_fw->size != mc_req_size) {
 790                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 791                                rdev->mc_fw->size, fw_name);
 792                         err = -EINVAL;
 793                 }
 794         }
 795
 796         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 797                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 798                 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 799                 if (err) {
 800                         pr_err("smc: error loading firmware \"%s\"\n", fw_name);
 801                         release_firmware(rdev->smc_fw);
 802                         rdev->smc_fw = NULL;
 803                         err = 0;
 804                 } else if (rdev->smc_fw->size != smc_req_size) {
 805                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 806                                rdev->mc_fw->size, fw_name);
 807                         err = -EINVAL;
 808                 }
 809         }
 810
 811 out:
 812         if (err) {
 813                 if (err != -EINVAL)
 814                         pr_err("ni_cp: Failed to load firmware \"%s\"\n",
 815                                fw_name);
 816                 release_firmware(rdev->pfp_fw);
 817                 rdev->pfp_fw = NULL;
 818                 release_firmware(rdev->me_fw);
 819                 rdev->me_fw = NULL;
 820                 release_firmware(rdev->rlc_fw);
 821                 rdev->rlc_fw = NULL;
 822                 release_firmware(rdev->mc_fw);
 823                 rdev->mc_fw = NULL;
 824         }
 825         return err;
 826 }
 827
 828 /**
 829  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 830  *
 831  * @rdev: radeon_device pointer
 832  * @reg: register offset in bytes
 833  * @val: register value
 834  *
 835  * Returns 0 for success or -EINVAL for an invalid register
 836  *
 837  */
 838 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 839                                      u32 reg, u32 *val)
 840 {
 841         switch (reg) {
 842         case GRBM_STATUS:
 843         case GRBM_STATUS_SE0:
 844         case GRBM_STATUS_SE1:
 845         case SRBM_STATUS:
 846         case SRBM_STATUS2:
 847         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 848         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 849         case UVD_STATUS:
 850                 *val = RREG32(reg);
 851                 return 0;
 852         default:
 853                 return -EINVAL;
 854         }
 855 }
 856
 857 int tn_get_temp(struct radeon_device *rdev)
 858 {
 859         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 860         int actual_temp = (temp / 8) - 49;
 861
 862         return actual_temp * 1000;
 863 }
 864
 865 /*
 866  * Core functions
 867  */
 868 static void cayman_gpu_init(struct radeon_device *rdev)
 869 {
 870         u32 gb_addr_config = 0;
 871         u32 mc_shared_chmap, mc_arb_ramcfg;
 872         u32 cgts_tcc_disable;
 873         u32 sx_debug_1;
 874         u32 smx_dc_ctl0;
 875         u32 cgts_sm_ctrl_reg;
 876         u32 hdp_host_path_cntl;
 877         u32 tmp;
 878         u32 disabled_rb_mask;
 879         int i, j;
 880
 881         switch (rdev->family) {
 882         case CHIP_CAYMAN:
 883                 rdev->config.cayman.max_shader_engines = 2;
 884                 rdev->config.cayman.max_pipes_per_simd = 4;
 885                 rdev->config.cayman.max_tile_pipes = 8;
 886                 rdev->config.cayman.max_simds_per_se = 12;
 887                 rdev->config.cayman.max_backends_per_se = 4;
 888                 rdev->config.cayman.max_texture_channel_caches = 8;
 889                 rdev->config.cayman.max_gprs = 256;
 890                 rdev->config.cayman.max_threads = 256;
 891                 rdev->config.cayman.max_gs_threads = 32;
 892                 rdev->config.cayman.max_stack_entries = 512;
 893                 rdev->config.cayman.sx_num_of_sets = 8;
 894                 rdev->config.cayman.sx_max_export_size = 256;
 895                 rdev->config.cayman.sx_max_export_pos_size = 64;
 896                 rdev->config.cayman.sx_max_export_smx_size = 192;
 897                 rdev->config.cayman.max_hw_contexts = 8;
 898                 rdev->config.cayman.sq_num_cf_insts = 2;
 899
 900                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 901                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 902                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 903                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 904                 break;
 905         case CHIP_ARUBA:
 906         default:
 907                 rdev->config.cayman.max_shader_engines = 1;
 908                 rdev->config.cayman.max_pipes_per_simd = 4;
 909                 rdev->config.cayman.max_tile_pipes = 2;
 910                 if ((rdev->pdev->device == 0x9900) ||
 911                     (rdev->pdev->device == 0x9901) ||
 912                     (rdev->pdev->device == 0x9905) ||
 913                     (rdev->pdev->device == 0x9906) ||
 914                     (rdev->pdev->device == 0x9907) ||
 915                     (rdev->pdev->device == 0x9908) ||
 916                     (rdev->pdev->device == 0x9909) ||
 917                     (rdev->pdev->device == 0x990B) ||
 918                     (rdev->pdev->device == 0x990C) ||
 919                     (rdev->pdev->device == 0x990F) ||
 920                     (rdev->pdev->device == 0x9910) ||
 921                     (rdev->pdev->device == 0x9917) ||
 922                     (rdev->pdev->device == 0x9999) ||
 923                     (rdev->pdev->device == 0x999C)) {
 924                         rdev->config.cayman.max_simds_per_se = 6;
 925                         rdev->config.cayman.max_backends_per_se = 2;
 926                         rdev->config.cayman.max_hw_contexts = 8;
 927                         rdev->config.cayman.sx_max_export_size = 256;
 928                         rdev->config.cayman.sx_max_export_pos_size = 64;
 929                         rdev->config.cayman.sx_max_export_smx_size = 192;
 930                 } else if ((rdev->pdev->device == 0x9903) ||
 931                            (rdev->pdev->device == 0x9904) ||
 932                            (rdev->pdev->device == 0x990A) ||
 933                            (rdev->pdev->device == 0x990D) ||
 934                            (rdev->pdev->device == 0x990E) ||
 935                            (rdev->pdev->device == 0x9913) ||
 936                            (rdev->pdev->device == 0x9918) ||
 937                            (rdev->pdev->device == 0x999D)) {
 938                         rdev->config.cayman.max_simds_per_se = 4;
 939                         rdev->config.cayman.max_backends_per_se = 2;
 940                         rdev->config.cayman.max_hw_contexts = 8;
 941                         rdev->config.cayman.sx_max_export_size = 256;
 942                         rdev->config.cayman.sx_max_export_pos_size = 64;
 943                         rdev->config.cayman.sx_max_export_smx_size = 192;
 944                 } else if ((rdev->pdev->device == 0x9919) ||
 945                            (rdev->pdev->device == 0x9990) ||
 946                            (rdev->pdev->device == 0x9991) ||
 947                            (rdev->pdev->device == 0x9994) ||
 948                            (rdev->pdev->device == 0x9995) ||
 949                            (rdev->pdev->device == 0x9996) ||
 950                            (rdev->pdev->device == 0x999A) ||
 951                            (rdev->pdev->device == 0x99A0)) {
 952                         rdev->config.cayman.max_simds_per_se = 3;
 953                         rdev->config.cayman.max_backends_per_se = 1;
 954                         rdev->config.cayman.max_hw_contexts = 4;
 955                         rdev->config.cayman.sx_max_export_size = 128;
 956                         rdev->config.cayman.sx_max_export_pos_size = 32;
 957                         rdev->config.cayman.sx_max_export_smx_size = 96;
 958                 } else {
 959                         rdev->config.cayman.max_simds_per_se = 2;
 960                         rdev->config.cayman.max_backends_per_se = 1;
 961                         rdev->config.cayman.max_hw_contexts = 4;
 962                         rdev->config.cayman.sx_max_export_size = 128;
 963                         rdev->config.cayman.sx_max_export_pos_size = 32;
 964                         rdev->config.cayman.sx_max_export_smx_size = 96;
 965                 }
 966                 rdev->config.cayman.max_texture_channel_caches = 2;
 967                 rdev->config.cayman.max_gprs = 256;
 968                 rdev->config.cayman.max_threads = 256;
 969                 rdev->config.cayman.max_gs_threads = 32;
 970                 rdev->config.cayman.max_stack_entries = 512;
 971                 rdev->config.cayman.sx_num_of_sets = 8;
 972                 rdev->config.cayman.sq_num_cf_insts = 2;
 973
 974                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 975                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 976                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 977                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 978                 break;
 979         }
 980
 981         /* Initialize HDP */
 982         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 983                 WREG32((0x2c14 + j), 0x00000000);
 984                 WREG32((0x2c18 + j), 0x00000000);
 985                 WREG32((0x2c1c + j), 0x00000000);
 986                 WREG32((0x2c20 + j), 0x00000000);
 987                 WREG32((0x2c24 + j), 0x00000000);
 988         }
 989
 990         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 991         WREG32(SRBM_INT_CNTL, 0x1);
 992         WREG32(SRBM_INT_ACK, 0x1);
 993
 994         evergreen_fix_pci_max_read_req_size(rdev);
 995
 996         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 997         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 998
 999         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1000         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1001         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1002                 rdev->config.cayman.mem_row_size_in_kb = 4;
1003         /* XXX use MC settings? */
1004         rdev->config.cayman.shader_engine_tile_size = 32;
1005         rdev->config.cayman.num_gpus = 1;
1006         rdev->config.cayman.multi_gpu_tile_size = 64;
1007
1008         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1009         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1010         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1011         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1012         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1013         rdev->config.cayman.num_shader_engines = tmp + 1;
1014         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1015         rdev->config.cayman.num_gpus = tmp + 1;
1016         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1017         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1018         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1019         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1020
1021
1022         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1023          * not have bank info, so create a custom tiling dword.
1024          * bits 3:0   num_pipes
1025          * bits 7:4   num_banks
1026          * bits 11:8  group_size
1027          * bits 15:12 row_size
1028          */
1029         rdev->config.cayman.tile_config = 0;
1030         switch (rdev->config.cayman.num_tile_pipes) {
1031         case 1:
1032         default:
1033                 rdev->config.cayman.tile_config |= (0 << 0);
1034                 break;
1035         case 2:
1036                 rdev->config.cayman.tile_config |= (1 << 0);
1037                 break;
1038         case 4:
1039                 rdev->config.cayman.tile_config |= (2 << 0);
1040                 break;
1041         case 8:
1042                 rdev->config.cayman.tile_config |= (3 << 0);
1043                 break;
1044         }
1045
1046         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1047         if (rdev->flags & RADEON_IS_IGP)
1048                 rdev->config.cayman.tile_config |= 1 << 4;
1049         else {
1050                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1051                 case 0: /* four banks */
1052                         rdev->config.cayman.tile_config |= 0 << 4;
1053                         break;
1054                 case 1: /* eight banks */
1055                         rdev->config.cayman.tile_config |= 1 << 4;
1056                         break;
1057                 case 2: /* sixteen banks */
1058                 default:
1059                         rdev->config.cayman.tile_config |= 2 << 4;
1060                         break;
1061                 }
1062         }
1063         rdev->config.cayman.tile_config |=
1064                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1065         rdev->config.cayman.tile_config |=
1066                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1067
1068         tmp = 0;
1069         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1070                 u32 rb_disable_bitmap;
1071
1072                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1073                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1074                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1075                 tmp <<= 4;
1076                 tmp |= rb_disable_bitmap;
1077         }
1078         /* enabled rb are just the one not disabled :) */
1079         disabled_rb_mask = tmp;
1080         tmp = 0;
1081         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1082                 tmp |= (1 << i);
1083         /* if all the backends are disabled, fix it up here */
1084         if ((disabled_rb_mask & tmp) == tmp) {
1085                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1086                         disabled_rb_mask &= ~(1 << i);
1087         }
1088
1089         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1090                 u32 simd_disable_bitmap;
1091
1092                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1093                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1094                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1095                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1096                 tmp <<= 16;
1097                 tmp |= simd_disable_bitmap;
1098         }
1099         rdev->config.cayman.active_simds = hweight32(~tmp);
1100
1101         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1102         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1103
1104         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1105         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1106         if (ASIC_IS_DCE6(rdev))
1107                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1108         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1109         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1110         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1111         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1112         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1113         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1114
1115         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1116             (rdev->flags & RADEON_IS_IGP)) {
1117                 if ((disabled_rb_mask & 3) == 2) {
1118                         /* RB1 disabled, RB0 enabled */
1119                         tmp = 0x00000000;
1120                 } else {
1121                         /* RB0 disabled, RB1 enabled */
1122                         tmp = 0x11111111;
1123                 }
1124         } else {
1125                 tmp = gb_addr_config & NUM_PIPES_MASK;
1126                 tmp = r6xx_remap_render_backend(rdev, tmp,
1127                                                 rdev->config.cayman.max_backends_per_se *
1128                                                 rdev->config.cayman.max_shader_engines,
1129                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1130         }
1131         WREG32(GB_BACKEND_MAP, tmp);
1132
1133         cgts_tcc_disable = 0xffff0000;
1134         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1135                 cgts_tcc_disable &= ~(1 << (16 + i));
1136         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1137         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1138         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1139         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1140
1141         /* reprogram the shader complex */
1142         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1143         for (i = 0; i < 16; i++)
1144                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1145         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1146
1147         /* set HW defaults for 3D engine */
1148         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1149
1150         sx_debug_1 = RREG32(SX_DEBUG_1);
1151         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1152         WREG32(SX_DEBUG_1, sx_debug_1);
1153
1154         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1155         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1156         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1157         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1158
1159         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1160
1161         /* need to be explicitly zero-ed */
1162         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1163         WREG32(SQ_LSTMP_RING_BASE, 0);
1164         WREG32(SQ_HSTMP_RING_BASE, 0);
1165         WREG32(SQ_ESTMP_RING_BASE, 0);
1166         WREG32(SQ_GSTMP_RING_BASE, 0);
1167         WREG32(SQ_VSTMP_RING_BASE, 0);
1168         WREG32(SQ_PSTMP_RING_BASE, 0);
1169
1170         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1171
1172         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1173                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1174                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1175
1176         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1177                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1178                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1179
1180
1181         WREG32(VGT_NUM_INSTANCES, 1);
1182
1183         WREG32(CP_PERFMON_CNTL, 0);
1184
1185         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1186                                   FETCH_FIFO_HIWATER(0x4) |
1187                                   DONE_FIFO_HIWATER(0xe0) |
1188                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1189
1190         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1191         WREG32(SQ_CONFIG, (VC_ENABLE |
1192                            EXPORT_SRC_C |
1193                            GFX_PRIO(0) |
1194                            CS1_PRIO(0) |
1195                            CS2_PRIO(1)));
1196         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1197
1198         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1199                                           FORCE_EOV_MAX_REZ_CNT(255)));
1200
1201         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1202                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1203
1204         WREG32(VGT_GS_VERTEX_REUSE, 16);
1205         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1206
1207         WREG32(CB_PERF_CTR0_SEL_0, 0);
1208         WREG32(CB_PERF_CTR0_SEL_1, 0);
1209         WREG32(CB_PERF_CTR1_SEL_0, 0);
1210         WREG32(CB_PERF_CTR1_SEL_1, 0);
1211         WREG32(CB_PERF_CTR2_SEL_0, 0);
1212         WREG32(CB_PERF_CTR2_SEL_1, 0);
1213         WREG32(CB_PERF_CTR3_SEL_0, 0);
1214         WREG32(CB_PERF_CTR3_SEL_1, 0);
1215
1216         tmp = RREG32(HDP_MISC_CNTL);
1217         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1218         WREG32(HDP_MISC_CNTL, tmp);
1219
1220         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1221         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1222
1223         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1224
1225         udelay(50);
1226
1227         /* set clockgating golden values on TN */
1228         if (rdev->family == CHIP_ARUBA) {
1229                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1230                 tmp &= ~0x00380000;
1231                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1232                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1233                 tmp &= ~0x0e000000;
1234                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1235         }
1236 }
1237
1238 /*
1239  * GART
1240  */
1241 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1242 {
1243         /* flush hdp cache */
1244         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1245
1246         /* bits 0-7 are the VM contexts0-7 */
1247         WREG32(VM_INVALIDATE_REQUEST, 1);
1248 }
1249
1250 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1251 {
1252         int i, r;
1253
1254         if (rdev->gart.robj == NULL) {
1255                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1256                 return -EINVAL;
1257         }
1258         r = radeon_gart_table_vram_pin(rdev);
1259         if (r)
1260                 return r;
1261         /* Setup TLB control */
1262         WREG32(MC_VM_MX_L1_TLB_CNTL,
1263                (0xA << 7) |
1264                ENABLE_L1_TLB |
1265                ENABLE_L1_FRAGMENT_PROCESSING |
1266                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1267                ENABLE_ADVANCED_DRIVER_MODEL |
1268                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1269         /* Setup L2 cache */
1270         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1271                ENABLE_L2_FRAGMENT_PROCESSING |
1272                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1273                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1274                EFFECTIVE_L2_QUEUE_SIZE(7) |
1275                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1276         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1277         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1278                BANK_SELECT(6) |
1279                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1280         /* setup context0 */
1281         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1282         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1283         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1284         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1285                         (u32)(rdev->dummy_page.addr >> 12));
1286         WREG32(VM_CONTEXT0_CNTL2, 0);
1287         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1288                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1289
1290         WREG32(0x15D4, 0);
1291         WREG32(0x15D8, 0);
1292         WREG32(0x15DC, 0);
1293
1294         /* empty context1-7 */
1295         /* Assign the pt base to something valid for now; the pts used for
1296          * the VMs are determined by the application and setup and assigned
1297          * on the fly in the vm part of radeon_gart.c
1298          */
1299         for (i = 1; i < 8; i++) {
1300                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1301                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1302                         rdev->vm_manager.max_pfn - 1);
1303                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1304                        rdev->vm_manager.saved_table_addr[i]);
1305         }
1306
1307         /* enable context1-7 */
1308         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1309                (u32)(rdev->dummy_page.addr >> 12));
1310         WREG32(VM_CONTEXT1_CNTL2, 4);
1311         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1312                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1313                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1314                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1315                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1316                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1317                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1318                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1319                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1320                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1321                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1322                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1323                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1324                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1325
1326         cayman_pcie_gart_tlb_flush(rdev);
1327         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1328                  (unsigned)(rdev->mc.gtt_size >> 20),
1329                  (unsigned long long)rdev->gart.table_addr);
1330         rdev->gart.ready = true;
1331         return 0;
1332 }
1333
1334 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1335 {
1336         unsigned i;
1337
1338         for (i = 1; i < 8; ++i) {
1339                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1340                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1341         }
1342
1343         /* Disable all tables */
1344         WREG32(VM_CONTEXT0_CNTL, 0);
1345         WREG32(VM_CONTEXT1_CNTL, 0);
1346         /* Setup TLB control */
1347         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1348                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1349                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1350         /* Setup L2 cache */
1351         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1352                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1353                EFFECTIVE_L2_QUEUE_SIZE(7) |
1354                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1355         WREG32(VM_L2_CNTL2, 0);
1356         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1357                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1358         radeon_gart_table_vram_unpin(rdev);
1359 }
1360
1361 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1362 {
1363         cayman_pcie_gart_disable(rdev);
1364         radeon_gart_table_vram_free(rdev);
1365         radeon_gart_fini(rdev);
1366 }
1367
1368 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1369                               int ring, u32 cp_int_cntl)
1370 {
1371         WREG32(SRBM_GFX_CNTL, RINGID(ring));
1372         WREG32(CP_INT_CNTL, cp_int_cntl);
1373 }
1374
1375 /*
1376  * CP.
1377  */
1378 void cayman_fence_ring_emit(struct radeon_device *rdev,
1379                             struct radeon_fence *fence)
1380 {
1381         struct radeon_ring *ring = &rdev->ring[fence->ring];
1382         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1383         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1384                 PACKET3_SH_ACTION_ENA;
1385
1386         /* flush read cache over gart for this vmid */
1387         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1388         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1389         radeon_ring_write(ring, 0xFFFFFFFF);
1390         radeon_ring_write(ring, 0);
1391         radeon_ring_write(ring, 10); /* poll interval */
1392         /* EVENT_WRITE_EOP - flush caches, send int */
1393         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1394         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1395         radeon_ring_write(ring, lower_32_bits(addr));
1396         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1397         radeon_ring_write(ring, fence->seq);
1398         radeon_ring_write(ring, 0);
1399 }
1400
1401 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1402 {
1403         struct radeon_ring *ring = &rdev->ring[ib->ring];
1404         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1405         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1406                 PACKET3_SH_ACTION_ENA;
1407
1408         /* set to DX10/11 mode */
1409         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1410         radeon_ring_write(ring, 1);
1411
1412         if (ring->rptr_save_reg) {
1413                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1414                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1415                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1416                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1417                 radeon_ring_write(ring, next_rptr);
1418         }
1419
1420         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1421         radeon_ring_write(ring,
1422 #ifdef __BIG_ENDIAN
1423                           (2 << 0) |
1424 #endif
1425                           (ib->gpu_addr & 0xFFFFFFFC));
1426         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1427         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1428
1429         /* flush read cache over gart for this vmid */
1430         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1431         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1432         radeon_ring_write(ring, 0xFFFFFFFF);
1433         radeon_ring_write(ring, 0);
1434         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1435 }
1436
1437 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1438 {
1439         if (enable)
1440                 WREG32(CP_ME_CNTL, 0);
1441         else {
1442                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1443                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1444                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1445                 WREG32(SCRATCH_UMSK, 0);
1446                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1447         }
1448 }
1449
1450 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1451                         struct radeon_ring *ring)
1452 {
1453         u32 rptr;
1454
1455         if (rdev->wb.enabled)
1456                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1457         else {
1458                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1459                         rptr = RREG32(CP_RB0_RPTR);
1460                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1461                         rptr = RREG32(CP_RB1_RPTR);
1462                 else
1463                         rptr = RREG32(CP_RB2_RPTR);
1464         }
1465
1466         return rptr;
1467 }
1468
1469 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1470                         struct radeon_ring *ring)
1471 {
1472         u32 wptr;
1473
1474         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1475                 wptr = RREG32(CP_RB0_WPTR);
1476         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1477                 wptr = RREG32(CP_RB1_WPTR);
1478         else
1479                 wptr = RREG32(CP_RB2_WPTR);
1480
1481         return wptr;
1482 }
1483
1484 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1485                          struct radeon_ring *ring)
1486 {
1487         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1488                 WREG32(CP_RB0_WPTR, ring->wptr);
1489                 (void)RREG32(CP_RB0_WPTR);
1490         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1491                 WREG32(CP_RB1_WPTR, ring->wptr);
1492                 (void)RREG32(CP_RB1_WPTR);
1493         } else {
1494                 WREG32(CP_RB2_WPTR, ring->wptr);
1495                 (void)RREG32(CP_RB2_WPTR);
1496         }
1497 }
1498
1499 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1500 {
1501         const __be32 *fw_data;
1502         int i;
1503
1504         if (!rdev->me_fw || !rdev->pfp_fw)
1505                 return -EINVAL;
1506
1507         cayman_cp_enable(rdev, false);
1508
1509         fw_data = (const __be32 *)rdev->pfp_fw->data;
1510         WREG32(CP_PFP_UCODE_ADDR, 0);
1511         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1512                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1513         WREG32(CP_PFP_UCODE_ADDR, 0);
1514
1515         fw_data = (const __be32 *)rdev->me_fw->data;
1516         WREG32(CP_ME_RAM_WADDR, 0);
1517         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1518                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1519
1520         WREG32(CP_PFP_UCODE_ADDR, 0);
1521         WREG32(CP_ME_RAM_WADDR, 0);
1522         WREG32(CP_ME_RAM_RADDR, 0);
1523         return 0;
1524 }
1525
1526 static int cayman_cp_start(struct radeon_device *rdev)
1527 {
1528         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1529         int r, i;
1530
1531         r = radeon_ring_lock(rdev, ring, 7);
1532         if (r) {
1533                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1534                 return r;
1535         }
1536         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1537         radeon_ring_write(ring, 0x1);
1538         radeon_ring_write(ring, 0x0);
1539         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1540         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1541         radeon_ring_write(ring, 0);
1542         radeon_ring_write(ring, 0);
1543         radeon_ring_unlock_commit(rdev, ring, false);
1544
1545         cayman_cp_enable(rdev, true);
1546
1547         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1548         if (r) {
1549                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1550                 return r;
1551         }
1552
1553         /* setup clear context state */
1554         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1555         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1556
1557         for (i = 0; i < cayman_default_size; i++)
1558                 radeon_ring_write(ring, cayman_default_state[i]);
1559
1560         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1561         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1562
1563         /* set clear context state */
1564         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1565         radeon_ring_write(ring, 0);
1566
1567         /* SQ_VTX_BASE_VTX_LOC */
1568         radeon_ring_write(ring, 0xc0026f00);
1569         radeon_ring_write(ring, 0x00000000);
1570         radeon_ring_write(ring, 0x00000000);
1571         radeon_ring_write(ring, 0x00000000);
1572
1573         /* Clear consts */
1574         radeon_ring_write(ring, 0xc0036f00);
1575         radeon_ring_write(ring, 0x00000bc4);
1576         radeon_ring_write(ring, 0xffffffff);
1577         radeon_ring_write(ring, 0xffffffff);
1578         radeon_ring_write(ring, 0xffffffff);
1579
1580         radeon_ring_write(ring, 0xc0026900);
1581         radeon_ring_write(ring, 0x00000316);
1582         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1583         radeon_ring_write(ring, 0x00000010); /*  */
1584
1585         radeon_ring_unlock_commit(rdev, ring, false);
1586
1587         /* XXX init other rings */
1588
1589         return 0;
1590 }
1591
1592 static void cayman_cp_fini(struct radeon_device *rdev)
1593 {
1594         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1595         cayman_cp_enable(rdev, false);
1596         radeon_ring_fini(rdev, ring);
1597         radeon_scratch_free(rdev, ring->rptr_save_reg);
1598 }
1599
1600 static int cayman_cp_resume(struct radeon_device *rdev)
1601 {
1602         static const int ridx[] = {
1603                 RADEON_RING_TYPE_GFX_INDEX,
1604                 CAYMAN_RING_TYPE_CP1_INDEX,
1605                 CAYMAN_RING_TYPE_CP2_INDEX
1606         };
1607         static const unsigned cp_rb_cntl[] = {
1608                 CP_RB0_CNTL,
1609                 CP_RB1_CNTL,
1610                 CP_RB2_CNTL,
1611         };
1612         static const unsigned cp_rb_rptr_addr[] = {
1613                 CP_RB0_RPTR_ADDR,
1614                 CP_RB1_RPTR_ADDR,
1615                 CP_RB2_RPTR_ADDR
1616         };
1617         static const unsigned cp_rb_rptr_addr_hi[] = {
1618                 CP_RB0_RPTR_ADDR_HI,
1619                 CP_RB1_RPTR_ADDR_HI,
1620                 CP_RB2_RPTR_ADDR_HI
1621         };
1622         static const unsigned cp_rb_base[] = {
1623                 CP_RB0_BASE,
1624                 CP_RB1_BASE,
1625                 CP_RB2_BASE
1626         };
1627         static const unsigned cp_rb_rptr[] = {
1628                 CP_RB0_RPTR,
1629                 CP_RB1_RPTR,
1630                 CP_RB2_RPTR
1631         };
1632         static const unsigned cp_rb_wptr[] = {
1633                 CP_RB0_WPTR,
1634                 CP_RB1_WPTR,
1635                 CP_RB2_WPTR
1636         };
1637         struct radeon_ring *ring;
1638         int i, r;
1639
1640         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1641         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1642                                  SOFT_RESET_PA |
1643                                  SOFT_RESET_SH |
1644                                  SOFT_RESET_VGT |
1645                                  SOFT_RESET_SPI |
1646                                  SOFT_RESET_SX));
1647         RREG32(GRBM_SOFT_RESET);
1648         mdelay(15);
1649         WREG32(GRBM_SOFT_RESET, 0);
1650         RREG32(GRBM_SOFT_RESET);
1651
1652         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1653         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1654
1655         /* Set the write pointer delay */
1656         WREG32(CP_RB_WPTR_DELAY, 0);
1657
1658         WREG32(CP_DEBUG, (1 << 27));
1659
1660         /* set the wb address whether it's enabled or not */
1661         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1662         WREG32(SCRATCH_UMSK, 0xff);
1663
1664         for (i = 0; i < 3; ++i) {
1665                 uint32_t rb_cntl;
1666                 uint64_t addr;
1667
1668                 /* Set ring buffer size */
1669                 ring = &rdev->ring[ridx[i]];
1670                 rb_cntl = order_base_2(ring->ring_size / 8);
1671                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1672 #ifdef __BIG_ENDIAN
1673                 rb_cntl |= BUF_SWAP_32BIT;
1674 #endif
1675                 WREG32(cp_rb_cntl[i], rb_cntl);
1676
1677                 /* set the wb address whether it's enabled or not */
1678                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1679                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1680                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1681         }
1682
1683         /* set the rb base addr, this causes an internal reset of ALL rings */
1684         for (i = 0; i < 3; ++i) {
1685                 ring = &rdev->ring[ridx[i]];
1686                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1687         }
1688
1689         for (i = 0; i < 3; ++i) {
1690                 /* Initialize the ring buffer's read and write pointers */
1691                 ring = &rdev->ring[ridx[i]];
1692                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1693
1694                 ring->wptr = 0;
1695                 WREG32(cp_rb_rptr[i], 0);
1696                 WREG32(cp_rb_wptr[i], ring->wptr);
1697
1698                 mdelay(1);
1699                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1700         }
1701
1702         /* start the rings */
1703         cayman_cp_start(rdev);
1704         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1705         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1706         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1707         /* this only test cp0 */
1708         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1709         if (r) {
1710                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1711                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1712                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1713                 return r;
1714         }
1715
1716         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1717                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1718
1719         return 0;
1720 }
1721
1722 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1723 {
1724         u32 reset_mask = 0;
1725         u32 tmp;
1726
1727         /* GRBM_STATUS */
1728         tmp = RREG32(GRBM_STATUS);
1729         if (tmp & (PA_BUSY | SC_BUSY |
1730                    SH_BUSY | SX_BUSY |
1731                    TA_BUSY | VGT_BUSY |
1732                    DB_BUSY | CB_BUSY |
1733                    GDS_BUSY | SPI_BUSY |
1734                    IA_BUSY | IA_BUSY_NO_DMA))
1735                 reset_mask |= RADEON_RESET_GFX;
1736
1737         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1738                    CP_BUSY | CP_COHERENCY_BUSY))
1739                 reset_mask |= RADEON_RESET_CP;
1740
1741         if (tmp & GRBM_EE_BUSY)
1742                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1743
1744         /* DMA_STATUS_REG 0 */
1745         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1746         if (!(tmp & DMA_IDLE))
1747                 reset_mask |= RADEON_RESET_DMA;
1748
1749         /* DMA_STATUS_REG 1 */
1750         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1751         if (!(tmp & DMA_IDLE))
1752                 reset_mask |= RADEON_RESET_DMA1;
1753
1754         /* SRBM_STATUS2 */
1755         tmp = RREG32(SRBM_STATUS2);
1756         if (tmp & DMA_BUSY)
1757                 reset_mask |= RADEON_RESET_DMA;
1758
1759         if (tmp & DMA1_BUSY)
1760                 reset_mask |= RADEON_RESET_DMA1;
1761
1762         /* SRBM_STATUS */
1763         tmp = RREG32(SRBM_STATUS);
1764         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1765                 reset_mask |= RADEON_RESET_RLC;
1766
1767         if (tmp & IH_BUSY)
1768                 reset_mask |= RADEON_RESET_IH;
1769
1770         if (tmp & SEM_BUSY)
1771                 reset_mask |= RADEON_RESET_SEM;
1772
1773         if (tmp & GRBM_RQ_PENDING)
1774                 reset_mask |= RADEON_RESET_GRBM;
1775
1776         if (tmp & VMC_BUSY)
1777                 reset_mask |= RADEON_RESET_VMC;
1778
1779         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1780                    MCC_BUSY | MCD_BUSY))
1781                 reset_mask |= RADEON_RESET_MC;
1782
1783         if (evergreen_is_display_hung(rdev))
1784                 reset_mask |= RADEON_RESET_DISPLAY;
1785
1786         /* VM_L2_STATUS */
1787         tmp = RREG32(VM_L2_STATUS);
1788         if (tmp & L2_BUSY)
1789                 reset_mask |= RADEON_RESET_VMC;
1790
1791         /* Skip MC reset as it's mostly likely not hung, just busy */
1792         if (reset_mask & RADEON_RESET_MC) {
1793                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1794                 reset_mask &= ~RADEON_RESET_MC;
1795         }
1796
1797         return reset_mask;
1798 }
1799
1800 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1801 {
1802         struct evergreen_mc_save save;
1803         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1804         u32 tmp;
1805
1806         if (reset_mask == 0)
1807                 return;
1808
1809         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1810
1811         evergreen_print_gpu_status_regs(rdev);
1812         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1813                  RREG32(0x14F8));
1814         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1815                  RREG32(0x14D8));
1816         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1817                  RREG32(0x14FC));
1818         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1819                  RREG32(0x14DC));
1820
1821         /* Disable CP parsing/prefetching */
1822         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1823
1824         if (reset_mask & RADEON_RESET_DMA) {
1825                 /* dma0 */
1826                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1827                 tmp &= ~DMA_RB_ENABLE;
1828                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1829         }
1830
1831         if (reset_mask & RADEON_RESET_DMA1) {
1832                 /* dma1 */
1833                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1834                 tmp &= ~DMA_RB_ENABLE;
1835                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1836         }
1837
1838         udelay(50);
1839
1840         evergreen_mc_stop(rdev, &save);
1841         if (evergreen_mc_wait_for_idle(rdev)) {
1842                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1843         }
1844
1845         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1846                 grbm_soft_reset = SOFT_RESET_CB |
1847                         SOFT_RESET_DB |
1848                         SOFT_RESET_GDS |
1849                         SOFT_RESET_PA |
1850                         SOFT_RESET_SC |
1851                         SOFT_RESET_SPI |
1852                         SOFT_RESET_SH |
1853                         SOFT_RESET_SX |
1854                         SOFT_RESET_TC |
1855                         SOFT_RESET_TA |
1856                         SOFT_RESET_VGT |
1857                         SOFT_RESET_IA;
1858         }
1859
1860         if (reset_mask & RADEON_RESET_CP) {
1861                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1862
1863                 srbm_soft_reset |= SOFT_RESET_GRBM;
1864         }
1865
1866         if (reset_mask & RADEON_RESET_DMA)
1867                 srbm_soft_reset |= SOFT_RESET_DMA;
1868
1869         if (reset_mask & RADEON_RESET_DMA1)
1870                 srbm_soft_reset |= SOFT_RESET_DMA1;
1871
1872         if (reset_mask & RADEON_RESET_DISPLAY)
1873                 srbm_soft_reset |= SOFT_RESET_DC;
1874
1875         if (reset_mask & RADEON_RESET_RLC)
1876                 srbm_soft_reset |= SOFT_RESET_RLC;
1877
1878         if (reset_mask & RADEON_RESET_SEM)
1879                 srbm_soft_reset |= SOFT_RESET_SEM;
1880
1881         if (reset_mask & RADEON_RESET_IH)
1882                 srbm_soft_reset |= SOFT_RESET_IH;
1883
1884         if (reset_mask & RADEON_RESET_GRBM)
1885                 srbm_soft_reset |= SOFT_RESET_GRBM;
1886
1887         if (reset_mask & RADEON_RESET_VMC)
1888                 srbm_soft_reset |= SOFT_RESET_VMC;
1889
1890         if (!(rdev->flags & RADEON_IS_IGP)) {
1891                 if (reset_mask & RADEON_RESET_MC)
1892                         srbm_soft_reset |= SOFT_RESET_MC;
1893         }
1894
1895         if (grbm_soft_reset) {
1896                 tmp = RREG32(GRBM_SOFT_RESET);
1897                 tmp |= grbm_soft_reset;
1898                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1899                 WREG32(GRBM_SOFT_RESET, tmp);
1900                 tmp = RREG32(GRBM_SOFT_RESET);
1901
1902                 udelay(50);
1903
1904                 tmp &= ~grbm_soft_reset;
1905                 WREG32(GRBM_SOFT_RESET, tmp);
1906                 tmp = RREG32(GRBM_SOFT_RESET);
1907         }
1908
1909         if (srbm_soft_reset) {
1910                 tmp = RREG32(SRBM_SOFT_RESET);
1911                 tmp |= srbm_soft_reset;
1912                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1913                 WREG32(SRBM_SOFT_RESET, tmp);
1914                 tmp = RREG32(SRBM_SOFT_RESET);
1915
1916                 udelay(50);
1917
1918                 tmp &= ~srbm_soft_reset;
1919                 WREG32(SRBM_SOFT_RESET, tmp);
1920                 tmp = RREG32(SRBM_SOFT_RESET);
1921         }
1922
1923         /* Wait a little for things to settle down */
1924         udelay(50);
1925
1926         evergreen_mc_resume(rdev, &save);
1927         udelay(50);
1928
1929         evergreen_print_gpu_status_regs(rdev);
1930 }
1931
1932 int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1933 {
1934         u32 reset_mask;
1935
1936         if (hard) {
1937                 evergreen_gpu_pci_config_reset(rdev);
1938                 return 0;
1939         }
1940
1941         reset_mask = cayman_gpu_check_soft_reset(rdev);
1942
1943         if (reset_mask)
1944                 r600_set_bios_scratch_engine_hung(rdev, true);
1945
1946         cayman_gpu_soft_reset(rdev, reset_mask);
1947
1948         reset_mask = cayman_gpu_check_soft_reset(rdev);
1949
1950         if (reset_mask)
1951                 evergreen_gpu_pci_config_reset(rdev);
1952
1953         r600_set_bios_scratch_engine_hung(rdev, false);
1954
1955         return 0;
1956 }
1957
1958 /**
1959  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1960  *
1961  * @rdev: radeon_device pointer
1962  * @ring: radeon_ring structure holding ring information
1963  *
1964  * Check if the GFX engine is locked up.
1965  * Returns true if the engine appears to be locked up, false if not.
1966  */
1967 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1968 {
1969         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1970
1971         if (!(reset_mask & (RADEON_RESET_GFX |
1972                             RADEON_RESET_COMPUTE |
1973                             RADEON_RESET_CP))) {
1974                 radeon_ring_lockup_update(rdev, ring);
1975                 return false;
1976         }
1977         return radeon_ring_test_lockup(rdev, ring);
1978 }
1979
1980 static void cayman_uvd_init(struct radeon_device *rdev)
1981 {
1982         int r;
1983
1984         if (!rdev->has_uvd)
1985                 return;
1986
1987         r = radeon_uvd_init(rdev);
1988         if (r) {
1989                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1990                 /*
1991                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1992                  * to early fails uvd_v2_2_resume() and thus nothing happens
1993                  * there. So it is pointless to try to go through that code
1994                  * hence why we disable uvd here.
1995                  */
1996                 rdev->has_uvd = 0;
1997                 return;
1998         }
1999         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
2000         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
2001 }
2002
2003 static void cayman_uvd_start(struct radeon_device *rdev)
2004 {
2005         int r;
2006
2007         if (!rdev->has_uvd)
2008                 return;
2009
2010         r = uvd_v2_2_resume(rdev);
2011         if (r) {
2012                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2013                 goto error;
2014         }
2015         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2016         if (r) {
2017                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2018                 goto error;
2019         }
2020         return;
2021
2022 error:
2023         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2024 }
2025
2026 static void cayman_uvd_resume(struct radeon_device *rdev)
2027 {
2028         struct radeon_ring *ring;
2029         int r;
2030
2031         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2032                 return;
2033
2034         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2035         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2036         if (r) {
2037                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2038                 return;
2039         }
2040         r = uvd_v1_0_init(rdev);
2041         if (r) {
2042                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2043                 return;
2044         }
2045 }
2046
2047 static void cayman_vce_init(struct radeon_device *rdev)
2048 {
2049         int r;
2050
2051         /* Only set for CHIP_ARUBA */
2052         if (!rdev->has_vce)
2053                 return;
2054
2055         r = radeon_vce_init(rdev);
2056         if (r) {
2057                 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2058                 /*
2059                  * At this point rdev->vce.vcpu_bo is NULL which trickles down
2060                  * to early fails cayman_vce_start() and thus nothing happens
2061                  * there. So it is pointless to try to go through that code
2062                  * hence why we disable vce here.
2063                  */
2064                 rdev->has_vce = 0;
2065                 return;
2066         }
2067         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2068         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2069         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2070         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2071 }
2072
2073 static void cayman_vce_start(struct radeon_device *rdev)
2074 {
2075         int r;
2076
2077         if (!rdev->has_vce)
2078                 return;
2079
2080         r = radeon_vce_resume(rdev);
2081         if (r) {
2082                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2083                 goto error;
2084         }
2085         r = vce_v1_0_resume(rdev);
2086         if (r) {
2087                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2088                 goto error;
2089         }
2090         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2091         if (r) {
2092                 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2093                 goto error;
2094         }
2095         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2096         if (r) {
2097                 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2098                 goto error;
2099         }
2100         return;
2101
2102 error:
2103         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2104         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2105 }
2106
2107 static void cayman_vce_resume(struct radeon_device *rdev)
2108 {
2109         struct radeon_ring *ring;
2110         int r;
2111
2112         if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2113                 return;
2114
2115         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2116         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2117         if (r) {
2118                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2119                 return;
2120         }
2121         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2122         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2123         if (r) {
2124                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2125                 return;
2126         }
2127         r = vce_v1_0_init(rdev);
2128         if (r) {
2129                 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2130                 return;
2131         }
2132 }
2133
2134 static int cayman_startup(struct radeon_device *rdev)
2135 {
2136         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2137         int r;
2138
2139         /* enable pcie gen2 link */
2140         evergreen_pcie_gen2_enable(rdev);
2141         /* enable aspm */
2142         evergreen_program_aspm(rdev);
2143
2144         /* scratch needs to be initialized before MC */
2145         r = r600_vram_scratch_init(rdev);
2146         if (r)
2147                 return r;
2148
2149         evergreen_mc_program(rdev);
2150
2151         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2152                 r = ni_mc_load_microcode(rdev);
2153                 if (r) {
2154                         DRM_ERROR("Failed to load MC firmware!\n");
2155                         return r;
2156                 }
2157         }
2158
2159         r = cayman_pcie_gart_enable(rdev);
2160         if (r)
2161                 return r;
2162         cayman_gpu_init(rdev);
2163
2164         /* allocate rlc buffers */
2165         if (rdev->flags & RADEON_IS_IGP) {
2166                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2167                 rdev->rlc.reg_list_size =
2168                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2169                 rdev->rlc.cs_data = cayman_cs_data;
2170                 r = sumo_rlc_init(rdev);
2171                 if (r) {
2172                         DRM_ERROR("Failed to init rlc BOs!\n");
2173                         return r;
2174                 }
2175         }
2176
2177         /* allocate wb buffer */
2178         r = radeon_wb_init(rdev);
2179         if (r)
2180                 return r;
2181
2182         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2183         if (r) {
2184                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2185                 return r;
2186         }
2187
2188         cayman_uvd_start(rdev);
2189         cayman_vce_start(rdev);
2190
2191         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2192         if (r) {
2193                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2194                 return r;
2195         }
2196
2197         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2198         if (r) {
2199                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2200                 return r;
2201         }
2202
2203         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2204         if (r) {
2205                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2206                 return r;
2207         }
2208
2209         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2210         if (r) {
2211                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2212                 return r;
2213         }
2214
2215         /* Enable IRQ */
2216         if (!rdev->irq.installed) {
2217                 r = radeon_irq_kms_init(rdev);
2218                 if (r)
2219                         return r;
2220         }
2221
2222         r = r600_irq_init(rdev);
2223         if (r) {
2224                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2225                 radeon_irq_kms_fini(rdev);
2226                 return r;
2227         }
2228         evergreen_irq_set(rdev);
2229
2230         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2231                              RADEON_CP_PACKET2);
2232         if (r)
2233                 return r;
2234
2235         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2236         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2237                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2238         if (r)
2239                 return r;
2240
2241         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2242         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2243                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2244         if (r)
2245                 return r;
2246
2247         r = cayman_cp_load_microcode(rdev);
2248         if (r)
2249                 return r;
2250         r = cayman_cp_resume(rdev);
2251         if (r)
2252                 return r;
2253
2254         r = cayman_dma_resume(rdev);
2255         if (r)
2256                 return r;
2257
2258         cayman_uvd_resume(rdev);
2259         cayman_vce_resume(rdev);
2260
2261         r = radeon_ib_pool_init(rdev);
2262         if (r) {
2263                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2264                 return r;
2265         }
2266
2267         r = radeon_vm_manager_init(rdev);
2268         if (r) {
2269                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2270                 return r;
2271         }
2272
2273         r = radeon_audio_init(rdev);
2274         if (r)
2275                 return r;
2276
2277         return 0;
2278 }
2279
2280 int cayman_resume(struct radeon_device *rdev)
2281 {
2282         int r;
2283
2284         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2285          * posting will perform necessary task to bring back GPU into good
2286          * shape.
2287          */
2288         /* post card */
2289         atom_asic_init(rdev->mode_info.atom_context);
2290
2291         /* init golden registers */
2292         ni_init_golden_registers(rdev);
2293
2294         if (rdev->pm.pm_method == PM_METHOD_DPM)
2295                 radeon_pm_resume(rdev);
2296
2297         rdev->accel_working = true;
2298         r = cayman_startup(rdev);
2299         if (r) {
2300                 DRM_ERROR("cayman startup failed on resume\n");
2301                 rdev->accel_working = false;
2302                 return r;
2303         }
2304         return r;
2305 }
2306
2307 int cayman_suspend(struct radeon_device *rdev)
2308 {
2309         radeon_pm_suspend(rdev);
2310         radeon_audio_fini(rdev);
2311         radeon_vm_manager_fini(rdev);
2312         cayman_cp_enable(rdev, false);
2313         cayman_dma_stop(rdev);
2314         if (rdev->has_uvd) {
2315                 uvd_v1_0_fini(rdev);
2316                 radeon_uvd_suspend(rdev);
2317         }
2318         evergreen_irq_suspend(rdev);
2319         radeon_wb_disable(rdev);
2320         cayman_pcie_gart_disable(rdev);
2321         return 0;
2322 }
2323
2324 /* Plan is to move initialization in that function and use
2325  * helper function so that radeon_device_init pretty much
2326  * do nothing more than calling asic specific function. This
2327  * should also allow to remove a bunch of callback function
2328  * like vram_info.
2329  */
2330 int cayman_init(struct radeon_device *rdev)
2331 {
2332         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2333         int r;
2334
2335         /* Read BIOS */
2336         if (!radeon_get_bios(rdev)) {
2337                 if (ASIC_IS_AVIVO(rdev))
2338                         return -EINVAL;
2339         }
2340         /* Must be an ATOMBIOS */
2341         if (!rdev->is_atom_bios) {
2342                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2343                 return -EINVAL;
2344         }
2345         r = radeon_atombios_init(rdev);
2346         if (r)
2347                 return r;
2348
2349         /* Post card if necessary */
2350         if (!radeon_card_posted(rdev)) {
2351                 if (!rdev->bios) {
2352                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2353                         return -EINVAL;
2354                 }
2355                 DRM_INFO("GPU not posted. posting now...\n");
2356                 atom_asic_init(rdev->mode_info.atom_context);
2357         }
2358         /* init golden registers */
2359         ni_init_golden_registers(rdev);
2360         /* Initialize scratch registers */
2361         r600_scratch_init(rdev);
2362         /* Initialize surface registers */
2363         radeon_surface_init(rdev);
2364         /* Initialize clocks */
2365         radeon_get_clock_info(rdev->ddev);
2366         /* Fence driver */
2367         r = radeon_fence_driver_init(rdev);
2368         if (r)
2369                 return r;
2370         /* initialize memory controller */
2371         r = evergreen_mc_init(rdev);
2372         if (r)
2373                 return r;
2374         /* Memory manager */
2375         r = radeon_bo_init(rdev);
2376         if (r)
2377                 return r;
2378
2379         if (rdev->flags & RADEON_IS_IGP) {
2380                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2381                         r = ni_init_microcode(rdev);
2382                         if (r) {
2383                                 DRM_ERROR("Failed to load firmware!\n");
2384                                 return r;
2385                         }
2386                 }
2387         } else {
2388                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2389                         r = ni_init_microcode(rdev);
2390                         if (r) {
2391                                 DRM_ERROR("Failed to load firmware!\n");
2392                                 return r;
2393                         }
2394                 }
2395         }
2396
2397         /* Initialize power management */
2398         radeon_pm_init(rdev);
2399
2400         ring->ring_obj = NULL;
2401         r600_ring_init(rdev, ring, 1024 * 1024);
2402
2403         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2404         ring->ring_obj = NULL;
2405         r600_ring_init(rdev, ring, 64 * 1024);
2406
2407         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2408         ring->ring_obj = NULL;
2409         r600_ring_init(rdev, ring, 64 * 1024);
2410
2411         cayman_uvd_init(rdev);
2412         cayman_vce_init(rdev);
2413
2414         rdev->ih.ring_obj = NULL;
2415         r600_ih_ring_init(rdev, 64 * 1024);
2416
2417         r = r600_pcie_gart_init(rdev);
2418         if (r)
2419                 return r;
2420
2421         rdev->accel_working = true;
2422         r = cayman_startup(rdev);
2423         if (r) {
2424                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2425                 cayman_cp_fini(rdev);
2426                 cayman_dma_fini(rdev);
2427                 r600_irq_fini(rdev);
2428                 if (rdev->flags & RADEON_IS_IGP)
2429                         sumo_rlc_fini(rdev);
2430                 radeon_wb_fini(rdev);
2431                 radeon_ib_pool_fini(rdev);
2432                 radeon_vm_manager_fini(rdev);
2433                 radeon_irq_kms_fini(rdev);
2434                 cayman_pcie_gart_fini(rdev);
2435                 rdev->accel_working = false;
2436         }
2437
2438         /* Don't start up if the MC ucode is missing.
2439          * The default clocks and voltages before the MC ucode
2440          * is loaded are not suffient for advanced operations.
2441          *
2442          * We can skip this check for TN, because there is no MC
2443          * ucode.
2444          */
2445         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2446                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2447                 return -EINVAL;
2448         }
2449
2450         return 0;
2451 }
2452
2453 void cayman_fini(struct radeon_device *rdev)
2454 {
2455         radeon_pm_fini(rdev);
2456         cayman_cp_fini(rdev);
2457         cayman_dma_fini(rdev);
2458         r600_irq_fini(rdev);
2459         if (rdev->flags & RADEON_IS_IGP)
2460                 sumo_rlc_fini(rdev);
2461         radeon_wb_fini(rdev);
2462         radeon_vm_manager_fini(rdev);
2463         radeon_ib_pool_fini(rdev);
2464         radeon_irq_kms_fini(rdev);
2465         uvd_v1_0_fini(rdev);
2466         radeon_uvd_fini(rdev);
2467         if (rdev->has_vce)
2468                 radeon_vce_fini(rdev);
2469         cayman_pcie_gart_fini(rdev);
2470         r600_vram_scratch_fini(rdev);
2471         radeon_gem_fini(rdev);
2472         radeon_fence_driver_fini(rdev);
2473         radeon_bo_fini(rdev);
2474         radeon_atombios_fini(rdev);
2475         kfree(rdev->bios);
2476         rdev->bios = NULL;
2477 }
2478
2479 /*
2480  * vm
2481  */
2482 int cayman_vm_init(struct radeon_device *rdev)
2483 {
2484         /* number of VMs */
2485         rdev->vm_manager.nvm = 8;
2486         /* base offset of vram pages */
2487         if (rdev->flags & RADEON_IS_IGP) {
2488                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2489                 tmp <<= 22;
2490                 rdev->vm_manager.vram_base_offset = tmp;
2491         } else
2492                 rdev->vm_manager.vram_base_offset = 0;
2493         return 0;
2494 }
2495
2496 void cayman_vm_fini(struct radeon_device *rdev)
2497 {
2498 }
2499
2500 /**
2501  * cayman_vm_decode_fault - print human readable fault info
2502  *
2503  * @rdev: radeon_device pointer
2504  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2505  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2506  *
2507  * Print human readable fault information (cayman/TN).
2508  */
2509 void cayman_vm_decode_fault(struct radeon_device *rdev,
2510                             u32 status, u32 addr)
2511 {
2512         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2513         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2514         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2515         char *block;
2516
2517         switch (mc_id) {
2518         case 32:
2519         case 16:
2520         case 96:
2521         case 80:
2522         case 160:
2523         case 144:
2524         case 224:
2525         case 208:
2526                 block = "CB";
2527                 break;
2528         case 33:
2529         case 17:
2530         case 97:
2531         case 81:
2532         case 161:
2533         case 145:
2534         case 225:
2535         case 209:
2536                 block = "CB_FMASK";
2537                 break;
2538         case 34:
2539         case 18:
2540         case 98:
2541         case 82:
2542         case 162:
2543         case 146:
2544         case 226:
2545         case 210:
2546                 block = "CB_CMASK";
2547                 break;
2548         case 35:
2549         case 19:
2550         case 99:
2551         case 83:
2552         case 163:
2553         case 147:
2554         case 227:
2555         case 211:
2556                 block = "CB_IMMED";
2557                 break;
2558         case 36:
2559         case 20:
2560         case 100:
2561         case 84:
2562         case 164:
2563         case 148:
2564         case 228:
2565         case 212:
2566                 block = "DB";
2567                 break;
2568         case 37:
2569         case 21:
2570         case 101:
2571         case 85:
2572         case 165:
2573         case 149:
2574         case 229:
2575         case 213:
2576                 block = "DB_HTILE";
2577                 break;
2578         case 38:
2579         case 22:
2580         case 102:
2581         case 86:
2582         case 166:
2583         case 150:
2584         case 230:
2585         case 214:
2586                 block = "SX";
2587                 break;
2588         case 39:
2589         case 23:
2590         case 103:
2591         case 87:
2592         case 167:
2593         case 151:
2594         case 231:
2595         case 215:
2596                 block = "DB_STEN";
2597                 break;
2598         case 40:
2599         case 24:
2600         case 104:
2601         case 88:
2602         case 232:
2603         case 216:
2604         case 168:
2605         case 152:
2606                 block = "TC_TFETCH";
2607                 break;
2608         case 41:
2609         case 25:
2610         case 105:
2611         case 89:
2612         case 233:
2613         case 217:
2614         case 169:
2615         case 153:
2616                 block = "TC_VFETCH";
2617                 break;
2618         case 42:
2619         case 26:
2620         case 106:
2621         case 90:
2622         case 234:
2623         case 218:
2624         case 170:
2625         case 154:
2626                 block = "VC";
2627                 break;
2628         case 112:
2629                 block = "CP";
2630                 break;
2631         case 113:
2632         case 114:
2633                 block = "SH";
2634                 break;
2635         case 115:
2636                 block = "VGT";
2637                 break;
2638         case 178:
2639                 block = "IH";
2640                 break;
2641         case 51:
2642                 block = "RLC";
2643                 break;
2644         case 55:
2645                 block = "DMA";
2646                 break;
2647         case 56:
2648                 block = "HDP";
2649                 break;
2650         default:
2651                 block = "unknown";
2652                 break;
2653         }
2654
2655         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2656                protections, vmid, addr,
2657                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2658                block, mc_id);
2659 }
2660
2661 /**
2662  * cayman_vm_flush - vm flush using the CP
2663  *
2664  * @rdev: radeon_device pointer
2665  *
2666  * Update the page table base and flush the VM TLB
2667  * using the CP (cayman-si).
2668  */
2669 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2670                      unsigned vm_id, uint64_t pd_addr)
2671 {
2672         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2673         radeon_ring_write(ring, pd_addr >> 12);
2674
2675         /* flush hdp cache */
2676         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2677         radeon_ring_write(ring, 0x1);
2678
2679         /* bits 0-7 are the VM contexts0-7 */
2680         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2681         radeon_ring_write(ring, 1 << vm_id);
2682
2683         /* wait for the invalidate to complete */
2684         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2685         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2686                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2687         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2688         radeon_ring_write(ring, 0);
2689         radeon_ring_write(ring, 0); /* ref */
2690         radeon_ring_write(ring, 0); /* mask */
2691         radeon_ring_write(ring, 0x20); /* poll interval */
2692
2693         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2694         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2695         radeon_ring_write(ring, 0x0);
2696 }
2697
2698 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2699 {
2700         struct atom_clock_dividers dividers;
2701         int r, i;
2702
2703         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2704                                            ecclk, false, &dividers);
2705         if (r)
2706                 return r;
2707
2708         for (i = 0; i < 100; i++) {
2709                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2710                         break;
2711                 mdelay(10);
2712         }
2713         if (i == 100)
2714                 return -ETIMEDOUT;
2715
2716         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2717
2718         for (i = 0; i < 100; i++) {
2719                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2720                         break;
2721                 mdelay(10);
2722         }
2723         if (i == 100)
2724                 return -ETIMEDOUT;
2725
2726         return 0;
2727 }