drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 /*
  40  * Indirect registers accessor
  41  */
  42 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  43 {
  44         unsigned long flags;
  45         u32 r;
  46
  47         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  48         WREG32(TN_SMC_IND_INDEX_0, (reg));
  49         r = RREG32(TN_SMC_IND_DATA_0);
  50         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  51         return r;
  52 }
  53
  54 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  55 {
  56         unsigned long flags;
  57
  58         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  59         WREG32(TN_SMC_IND_INDEX_0, (reg));
  60         WREG32(TN_SMC_IND_DATA_0, (v));
  61         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  62 }
  63
  64 static const u32 tn_rlc_save_restore_register_list[] =
  65 {
  66         0x98fc,
  67         0x98f0,
  68         0x9834,
  69         0x9838,
  70         0x9870,
  71         0x9874,
  72         0x8a14,
  73         0x8b24,
  74         0x8bcc,
  75         0x8b10,
  76         0x8c30,
  77         0x8d00,
  78         0x8d04,
  79         0x8c00,
  80         0x8c04,
  81         0x8c10,
  82         0x8c14,
  83         0x8d8c,
  84         0x8cf0,
  85         0x8e38,
  86         0x9508,
  87         0x9688,
  88         0x9608,
  89         0x960c,
  90         0x9610,
  91         0x9614,
  92         0x88c4,
  93         0x8978,
  94         0x88d4,
  95         0x900c,
  96         0x9100,
  97         0x913c,
  98         0x90e8,
  99         0x9354,
 100         0xa008,
 101         0x98f8,
 102         0x9148,
 103         0x914c,
 104         0x3f94,
 105         0x98f4,
 106         0x9b7c,
 107         0x3f8c,
 108         0x8950,
 109         0x8954,
 110         0x8a18,
 111         0x8b28,
 112         0x9144,
 113         0x3f90,
 114         0x915c,
 115         0x9160,
 116         0x9178,
 117         0x917c,
 118         0x9180,
 119         0x918c,
 120         0x9190,
 121         0x9194,
 122         0x9198,
 123         0x919c,
 124         0x91a8,
 125         0x91ac,
 126         0x91b0,
 127         0x91b4,
 128         0x91b8,
 129         0x91c4,
 130         0x91c8,
 131         0x91cc,
 132         0x91d0,
 133         0x91d4,
 134         0x91e0,
 135         0x91e4,
 136         0x91ec,
 137         0x91f0,
 138         0x91f4,
 139         0x9200,
 140         0x9204,
 141         0x929c,
 142         0x8030,
 143         0x9150,
 144         0x9a60,
 145         0x920c,
 146         0x9210,
 147         0x9228,
 148         0x922c,
 149         0x9244,
 150         0x9248,
 151         0x91e8,
 152         0x9294,
 153         0x9208,
 154         0x9224,
 155         0x9240,
 156         0x9220,
 157         0x923c,
 158         0x9258,
 159         0x9744,
 160         0xa200,
 161         0xa204,
 162         0xa208,
 163         0xa20c,
 164         0x8d58,
 165         0x9030,
 166         0x9034,
 167         0x9038,
 168         0x903c,
 169         0x9040,
 170         0x9654,
 171         0x897c,
 172         0xa210,
 173         0xa214,
 174         0x9868,
 175         0xa02c,
 176         0x9664,
 177         0x9698,
 178         0x949c,
 179         0x8e10,
 180         0x8e18,
 181         0x8c50,
 182         0x8c58,
 183         0x8c60,
 184         0x8c68,
 185         0x89b4,
 186         0x9830,
 187         0x802c,
 188 };
 189
 190 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 191 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 192 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 193 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 194 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 195 extern void evergreen_mc_program(struct radeon_device *rdev);
 196 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 197 extern int evergreen_mc_init(struct radeon_device *rdev);
 198 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 199 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 200 extern void evergreen_program_aspm(struct radeon_device *rdev);
 201 extern void sumo_rlc_fini(struct radeon_device *rdev);
 202 extern int sumo_rlc_init(struct radeon_device *rdev);
 203 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 204
 205 /* Firmware Names */
 206 /*(DEBLOBBED)*/
 207
 208
 209 static const u32 cayman_golden_registers2[] =
 210 {
 211         0x3e5c, 0xffffffff, 0x00000000,
 212         0x3e48, 0xffffffff, 0x00000000,
 213         0x3e4c, 0xffffffff, 0x00000000,
 214         0x3e64, 0xffffffff, 0x00000000,
 215         0x3e50, 0xffffffff, 0x00000000,
 216         0x3e60, 0xffffffff, 0x00000000
 217 };
 218
 219 static const u32 cayman_golden_registers[] =
 220 {
 221         0x5eb4, 0xffffffff, 0x00000002,
 222         0x5e78, 0x8f311ff1, 0x001000f0,
 223         0x3f90, 0xffff0000, 0xff000000,
 224         0x9148, 0xffff0000, 0xff000000,
 225         0x3f94, 0xffff0000, 0xff000000,
 226         0x914c, 0xffff0000, 0xff000000,
 227         0xc78, 0x00000080, 0x00000080,
 228         0xbd4, 0x70073777, 0x00011003,
 229         0xd02c, 0xbfffff1f, 0x08421000,
 230         0xd0b8, 0x73773777, 0x02011003,
 231         0x5bc0, 0x00200000, 0x50100000,
 232         0x98f8, 0x33773777, 0x02011003,
 233         0x98fc, 0xffffffff, 0x76541032,
 234         0x7030, 0x31000311, 0x00000011,
 235         0x2f48, 0x33773777, 0x42010001,
 236         0x6b28, 0x00000010, 0x00000012,
 237         0x7728, 0x00000010, 0x00000012,
 238         0x10328, 0x00000010, 0x00000012,
 239         0x10f28, 0x00000010, 0x00000012,
 240         0x11b28, 0x00000010, 0x00000012,
 241         0x12728, 0x00000010, 0x00000012,
 242         0x240c, 0x000007ff, 0x00000000,
 243         0x8a14, 0xf000001f, 0x00000007,
 244         0x8b24, 0x3fff3fff, 0x00ff0fff,
 245         0x8b10, 0x0000ff0f, 0x00000000,
 246         0x28a4c, 0x07ffffff, 0x06000000,
 247         0x10c, 0x00000001, 0x00010003,
 248         0xa02c, 0xffffffff, 0x0000009b,
 249         0x913c, 0x0000010f, 0x01000100,
 250         0x8c04, 0xf8ff00ff, 0x40600060,
 251         0x28350, 0x00000f01, 0x00000000,
 252         0x9508, 0x3700001f, 0x00000002,
 253         0x960c, 0xffffffff, 0x54763210,
 254         0x88c4, 0x001f3ae3, 0x00000082,
 255         0x88d0, 0xffffffff, 0x0f40df40,
 256         0x88d4, 0x0000001f, 0x00000010,
 257         0x8974, 0xffffffff, 0x00000000
 258 };
 259
 260 static const u32 dvst_golden_registers2[] =
 261 {
 262         0x8f8, 0xffffffff, 0,
 263         0x8fc, 0x00380000, 0,
 264         0x8f8, 0xffffffff, 1,
 265         0x8fc, 0x0e000000, 0
 266 };
 267
 268 static const u32 dvst_golden_registers[] =
 269 {
 270         0x690, 0x3fff3fff, 0x20c00033,
 271         0x918c, 0x0fff0fff, 0x00010006,
 272         0x91a8, 0x0fff0fff, 0x00010006,
 273         0x9150, 0xffffdfff, 0x6e944040,
 274         0x917c, 0x0fff0fff, 0x00030002,
 275         0x9198, 0x0fff0fff, 0x00030002,
 276         0x915c, 0x0fff0fff, 0x00010000,
 277         0x3f90, 0xffff0001, 0xff000000,
 278         0x9178, 0x0fff0fff, 0x00070000,
 279         0x9194, 0x0fff0fff, 0x00070000,
 280         0x9148, 0xffff0001, 0xff000000,
 281         0x9190, 0x0fff0fff, 0x00090008,
 282         0x91ac, 0x0fff0fff, 0x00090008,
 283         0x3f94, 0xffff0000, 0xff000000,
 284         0x914c, 0xffff0000, 0xff000000,
 285         0x929c, 0x00000fff, 0x00000001,
 286         0x55e4, 0xff607fff, 0xfc000100,
 287         0x8a18, 0xff000fff, 0x00000100,
 288         0x8b28, 0xff000fff, 0x00000100,
 289         0x9144, 0xfffc0fff, 0x00000100,
 290         0x6ed8, 0x00010101, 0x00010000,
 291         0x9830, 0xffffffff, 0x00000000,
 292         0x9834, 0xf00fffff, 0x00000400,
 293         0x9838, 0xfffffffe, 0x00000000,
 294         0xd0c0, 0xff000fff, 0x00000100,
 295         0xd02c, 0xbfffff1f, 0x08421000,
 296         0xd0b8, 0x73773777, 0x12010001,
 297         0x5bb0, 0x000000f0, 0x00000070,
 298         0x98f8, 0x73773777, 0x12010001,
 299         0x98fc, 0xffffffff, 0x00000010,
 300         0x9b7c, 0x00ff0000, 0x00fc0000,
 301         0x8030, 0x00001f0f, 0x0000100a,
 302         0x2f48, 0x73773777, 0x12010001,
 303         0x2408, 0x00030000, 0x000c007f,
 304         0x8a14, 0xf000003f, 0x00000007,
 305         0x8b24, 0x3fff3fff, 0x00ff0fff,
 306         0x8b10, 0x0000ff0f, 0x00000000,
 307         0x28a4c, 0x07ffffff, 0x06000000,
 308         0x4d8, 0x00000fff, 0x00000100,
 309         0xa008, 0xffffffff, 0x00010000,
 310         0x913c, 0xffff03ff, 0x01000100,
 311         0x8c00, 0x000000ff, 0x00000003,
 312         0x8c04, 0xf8ff00ff, 0x40600060,
 313         0x8cf0, 0x1fff1fff, 0x08e00410,
 314         0x28350, 0x00000f01, 0x00000000,
 315         0x9508, 0xf700071f, 0x00000002,
 316         0x960c, 0xffffffff, 0x54763210,
 317         0x20ef8, 0x01ff01ff, 0x00000002,
 318         0x20e98, 0xfffffbff, 0x00200000,
 319         0x2015c, 0xffffffff, 0x00000f40,
 320         0x88c4, 0x001f3ae3, 0x00000082,
 321         0x8978, 0x3fffffff, 0x04050140,
 322         0x88d4, 0x0000001f, 0x00000010,
 323         0x8974, 0xffffffff, 0x00000000
 324 };
 325
 326 static const u32 scrapper_golden_registers[] =
 327 {
 328         0x690, 0x3fff3fff, 0x20c00033,
 329         0x918c, 0x0fff0fff, 0x00010006,
 330         0x918c, 0x0fff0fff, 0x00010006,
 331         0x91a8, 0x0fff0fff, 0x00010006,
 332         0x91a8, 0x0fff0fff, 0x00010006,
 333         0x9150, 0xffffdfff, 0x6e944040,
 334         0x9150, 0xffffdfff, 0x6e944040,
 335         0x917c, 0x0fff0fff, 0x00030002,
 336         0x917c, 0x0fff0fff, 0x00030002,
 337         0x9198, 0x0fff0fff, 0x00030002,
 338         0x9198, 0x0fff0fff, 0x00030002,
 339         0x915c, 0x0fff0fff, 0x00010000,
 340         0x915c, 0x0fff0fff, 0x00010000,
 341         0x3f90, 0xffff0001, 0xff000000,
 342         0x3f90, 0xffff0001, 0xff000000,
 343         0x9178, 0x0fff0fff, 0x00070000,
 344         0x9178, 0x0fff0fff, 0x00070000,
 345         0x9194, 0x0fff0fff, 0x00070000,
 346         0x9194, 0x0fff0fff, 0x00070000,
 347         0x9148, 0xffff0001, 0xff000000,
 348         0x9148, 0xffff0001, 0xff000000,
 349         0x9190, 0x0fff0fff, 0x00090008,
 350         0x9190, 0x0fff0fff, 0x00090008,
 351         0x91ac, 0x0fff0fff, 0x00090008,
 352         0x91ac, 0x0fff0fff, 0x00090008,
 353         0x3f94, 0xffff0000, 0xff000000,
 354         0x3f94, 0xffff0000, 0xff000000,
 355         0x914c, 0xffff0000, 0xff000000,
 356         0x914c, 0xffff0000, 0xff000000,
 357         0x929c, 0x00000fff, 0x00000001,
 358         0x929c, 0x00000fff, 0x00000001,
 359         0x55e4, 0xff607fff, 0xfc000100,
 360         0x8a18, 0xff000fff, 0x00000100,
 361         0x8a18, 0xff000fff, 0x00000100,
 362         0x8b28, 0xff000fff, 0x00000100,
 363         0x8b28, 0xff000fff, 0x00000100,
 364         0x9144, 0xfffc0fff, 0x00000100,
 365         0x9144, 0xfffc0fff, 0x00000100,
 366         0x6ed8, 0x00010101, 0x00010000,
 367         0x9830, 0xffffffff, 0x00000000,
 368         0x9830, 0xffffffff, 0x00000000,
 369         0x9834, 0xf00fffff, 0x00000400,
 370         0x9834, 0xf00fffff, 0x00000400,
 371         0x9838, 0xfffffffe, 0x00000000,
 372         0x9838, 0xfffffffe, 0x00000000,
 373         0xd0c0, 0xff000fff, 0x00000100,
 374         0xd02c, 0xbfffff1f, 0x08421000,
 375         0xd02c, 0xbfffff1f, 0x08421000,
 376         0xd0b8, 0x73773777, 0x12010001,
 377         0xd0b8, 0x73773777, 0x12010001,
 378         0x5bb0, 0x000000f0, 0x00000070,
 379         0x98f8, 0x73773777, 0x12010001,
 380         0x98f8, 0x73773777, 0x12010001,
 381         0x98fc, 0xffffffff, 0x00000010,
 382         0x98fc, 0xffffffff, 0x00000010,
 383         0x9b7c, 0x00ff0000, 0x00fc0000,
 384         0x9b7c, 0x00ff0000, 0x00fc0000,
 385         0x8030, 0x00001f0f, 0x0000100a,
 386         0x8030, 0x00001f0f, 0x0000100a,
 387         0x2f48, 0x73773777, 0x12010001,
 388         0x2f48, 0x73773777, 0x12010001,
 389         0x2408, 0x00030000, 0x000c007f,
 390         0x8a14, 0xf000003f, 0x00000007,
 391         0x8a14, 0xf000003f, 0x00000007,
 392         0x8b24, 0x3fff3fff, 0x00ff0fff,
 393         0x8b24, 0x3fff3fff, 0x00ff0fff,
 394         0x8b10, 0x0000ff0f, 0x00000000,
 395         0x8b10, 0x0000ff0f, 0x00000000,
 396         0x28a4c, 0x07ffffff, 0x06000000,
 397         0x28a4c, 0x07ffffff, 0x06000000,
 398         0x4d8, 0x00000fff, 0x00000100,
 399         0x4d8, 0x00000fff, 0x00000100,
 400         0xa008, 0xffffffff, 0x00010000,
 401         0xa008, 0xffffffff, 0x00010000,
 402         0x913c, 0xffff03ff, 0x01000100,
 403         0x913c, 0xffff03ff, 0x01000100,
 404         0x90e8, 0x001fffff, 0x010400c0,
 405         0x8c00, 0x000000ff, 0x00000003,
 406         0x8c00, 0x000000ff, 0x00000003,
 407         0x8c04, 0xf8ff00ff, 0x40600060,
 408         0x8c04, 0xf8ff00ff, 0x40600060,
 409         0x8c30, 0x0000000f, 0x00040005,
 410         0x8cf0, 0x1fff1fff, 0x08e00410,
 411         0x8cf0, 0x1fff1fff, 0x08e00410,
 412         0x900c, 0x00ffffff, 0x0017071f,
 413         0x28350, 0x00000f01, 0x00000000,
 414         0x28350, 0x00000f01, 0x00000000,
 415         0x9508, 0xf700071f, 0x00000002,
 416         0x9508, 0xf700071f, 0x00000002,
 417         0x9688, 0x00300000, 0x0017000f,
 418         0x960c, 0xffffffff, 0x54763210,
 419         0x960c, 0xffffffff, 0x54763210,
 420         0x20ef8, 0x01ff01ff, 0x00000002,
 421         0x20e98, 0xfffffbff, 0x00200000,
 422         0x2015c, 0xffffffff, 0x00000f40,
 423         0x88c4, 0x001f3ae3, 0x00000082,
 424         0x88c4, 0x001f3ae3, 0x00000082,
 425         0x8978, 0x3fffffff, 0x04050140,
 426         0x8978, 0x3fffffff, 0x04050140,
 427         0x88d4, 0x0000001f, 0x00000010,
 428         0x88d4, 0x0000001f, 0x00000010,
 429         0x8974, 0xffffffff, 0x00000000,
 430         0x8974, 0xffffffff, 0x00000000
 431 };
 432
 433 static void ni_init_golden_registers(struct radeon_device *rdev)
 434 {
 435         switch (rdev->family) {
 436         case CHIP_CAYMAN:
 437                 radeon_program_register_sequence(rdev,
 438                                                  cayman_golden_registers,
 439                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 440                 radeon_program_register_sequence(rdev,
 441                                                  cayman_golden_registers2,
 442                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 443                 break;
 444         case CHIP_ARUBA:
 445                 if ((rdev->pdev->device == 0x9900) ||
 446                     (rdev->pdev->device == 0x9901) ||
 447                     (rdev->pdev->device == 0x9903) ||
 448                     (rdev->pdev->device == 0x9904) ||
 449                     (rdev->pdev->device == 0x9905) ||
 450                     (rdev->pdev->device == 0x9906) ||
 451                     (rdev->pdev->device == 0x9907) ||
 452                     (rdev->pdev->device == 0x9908) ||
 453                     (rdev->pdev->device == 0x9909) ||
 454                     (rdev->pdev->device == 0x990A) ||
 455                     (rdev->pdev->device == 0x990B) ||
 456                     (rdev->pdev->device == 0x990C) ||
 457                     (rdev->pdev->device == 0x990D) ||
 458                     (rdev->pdev->device == 0x990E) ||
 459                     (rdev->pdev->device == 0x990F) ||
 460                     (rdev->pdev->device == 0x9910) ||
 461                     (rdev->pdev->device == 0x9913) ||
 462                     (rdev->pdev->device == 0x9917) ||
 463                     (rdev->pdev->device == 0x9918)) {
 464                         radeon_program_register_sequence(rdev,
 465                                                          dvst_golden_registers,
 466                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 467                         radeon_program_register_sequence(rdev,
 468                                                          dvst_golden_registers2,
 469                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 470                 } else {
 471                         radeon_program_register_sequence(rdev,
 472                                                          scrapper_golden_registers,
 473                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 474                         radeon_program_register_sequence(rdev,
 475                                                          dvst_golden_registers2,
 476                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 477                 }
 478                 break;
 479         default:
 480                 break;
 481         }
 482 }
 483
 484 #define BTC_IO_MC_REGS_SIZE 29
 485
 486 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 487         {0x00000077, 0xff010100},
 488         {0x00000078, 0x00000000},
 489         {0x00000079, 0x00001434},
 490         {0x0000007a, 0xcc08ec08},
 491         {0x0000007b, 0x00040000},
 492         {0x0000007c, 0x000080c0},
 493         {0x0000007d, 0x09000000},
 494         {0x0000007e, 0x00210404},
 495         {0x00000081, 0x08a8e800},
 496         {0x00000082, 0x00030444},
 497         {0x00000083, 0x00000000},
 498         {0x00000085, 0x00000001},
 499         {0x00000086, 0x00000002},
 500         {0x00000087, 0x48490000},
 501         {0x00000088, 0x20244647},
 502         {0x00000089, 0x00000005},
 503         {0x0000008b, 0x66030000},
 504         {0x0000008c, 0x00006603},
 505         {0x0000008d, 0x00000100},
 506         {0x0000008f, 0x00001c0a},
 507         {0x00000090, 0xff000001},
 508         {0x00000094, 0x00101101},
 509         {0x00000095, 0x00000fff},
 510         {0x00000096, 0x00116fff},
 511         {0x00000097, 0x60010000},
 512         {0x00000098, 0x10010000},
 513         {0x00000099, 0x00006000},
 514         {0x0000009a, 0x00001000},
 515         {0x0000009f, 0x00946a00}
 516 };
 517
 518 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 519         {0x00000077, 0xff010100},
 520         {0x00000078, 0x00000000},
 521         {0x00000079, 0x00001434},
 522         {0x0000007a, 0xcc08ec08},
 523         {0x0000007b, 0x00040000},
 524         {0x0000007c, 0x000080c0},
 525         {0x0000007d, 0x09000000},
 526         {0x0000007e, 0x00210404},
 527         {0x00000081, 0x08a8e800},
 528         {0x00000082, 0x00030444},
 529         {0x00000083, 0x00000000},
 530         {0x00000085, 0x00000001},
 531         {0x00000086, 0x00000002},
 532         {0x00000087, 0x48490000},
 533         {0x00000088, 0x20244647},
 534         {0x00000089, 0x00000005},
 535         {0x0000008b, 0x66030000},
 536         {0x0000008c, 0x00006603},
 537         {0x0000008d, 0x00000100},
 538         {0x0000008f, 0x00001c0a},
 539         {0x00000090, 0xff000001},
 540         {0x00000094, 0x00101101},
 541         {0x00000095, 0x00000fff},
 542         {0x00000096, 0x00116fff},
 543         {0x00000097, 0x60010000},
 544         {0x00000098, 0x10010000},
 545         {0x00000099, 0x00006000},
 546         {0x0000009a, 0x00001000},
 547         {0x0000009f, 0x00936a00}
 548 };
 549
 550 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 551         {0x00000077, 0xff010100},
 552         {0x00000078, 0x00000000},
 553         {0x00000079, 0x00001434},
 554         {0x0000007a, 0xcc08ec08},
 555         {0x0000007b, 0x00040000},
 556         {0x0000007c, 0x000080c0},
 557         {0x0000007d, 0x09000000},
 558         {0x0000007e, 0x00210404},
 559         {0x00000081, 0x08a8e800},
 560         {0x00000082, 0x00030444},
 561         {0x00000083, 0x00000000},
 562         {0x00000085, 0x00000001},
 563         {0x00000086, 0x00000002},
 564         {0x00000087, 0x48490000},
 565         {0x00000088, 0x20244647},
 566         {0x00000089, 0x00000005},
 567         {0x0000008b, 0x66030000},
 568         {0x0000008c, 0x00006603},
 569         {0x0000008d, 0x00000100},
 570         {0x0000008f, 0x00001c0a},
 571         {0x00000090, 0xff000001},
 572         {0x00000094, 0x00101101},
 573         {0x00000095, 0x00000fff},
 574         {0x00000096, 0x00116fff},
 575         {0x00000097, 0x60010000},
 576         {0x00000098, 0x10010000},
 577         {0x00000099, 0x00006000},
 578         {0x0000009a, 0x00001000},
 579         {0x0000009f, 0x00916a00}
 580 };
 581
 582 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 583         {0x00000077, 0xff010100},
 584         {0x00000078, 0x00000000},
 585         {0x00000079, 0x00001434},
 586         {0x0000007a, 0xcc08ec08},
 587         {0x0000007b, 0x00040000},
 588         {0x0000007c, 0x000080c0},
 589         {0x0000007d, 0x09000000},
 590         {0x0000007e, 0x00210404},
 591         {0x00000081, 0x08a8e800},
 592         {0x00000082, 0x00030444},
 593         {0x00000083, 0x00000000},
 594         {0x00000085, 0x00000001},
 595         {0x00000086, 0x00000002},
 596         {0x00000087, 0x48490000},
 597         {0x00000088, 0x20244647},
 598         {0x00000089, 0x00000005},
 599         {0x0000008b, 0x66030000},
 600         {0x0000008c, 0x00006603},
 601         {0x0000008d, 0x00000100},
 602         {0x0000008f, 0x00001c0a},
 603         {0x00000090, 0xff000001},
 604         {0x00000094, 0x00101101},
 605         {0x00000095, 0x00000fff},
 606         {0x00000096, 0x00116fff},
 607         {0x00000097, 0x60010000},
 608         {0x00000098, 0x10010000},
 609         {0x00000099, 0x00006000},
 610         {0x0000009a, 0x00001000},
 611         {0x0000009f, 0x00976b00}
 612 };
 613
 614 int ni_mc_load_microcode(struct radeon_device *rdev)
 615 {
 616         const __be32 *fw_data;
 617         u32 mem_type, running, blackout = 0;
 618         u32 *io_mc_regs;
 619         int i, ucode_size, regs_size;
 620
 621         if (!rdev->mc_fw)
 622                 return -EINVAL;
 623
 624         switch (rdev->family) {
 625         case CHIP_BARTS:
 626                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 627                 ucode_size = BTC_MC_UCODE_SIZE;
 628                 regs_size = BTC_IO_MC_REGS_SIZE;
 629                 break;
 630         case CHIP_TURKS:
 631                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 632                 ucode_size = BTC_MC_UCODE_SIZE;
 633                 regs_size = BTC_IO_MC_REGS_SIZE;
 634                 break;
 635         case CHIP_CAICOS:
 636         default:
 637                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 638                 ucode_size = BTC_MC_UCODE_SIZE;
 639                 regs_size = BTC_IO_MC_REGS_SIZE;
 640                 break;
 641         case CHIP_CAYMAN:
 642                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 643                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 644                 regs_size = BTC_IO_MC_REGS_SIZE;
 645                 break;
 646         }
 647
 648         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 649         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 650
 651         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 652                 if (running) {
 653                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 654                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 655                 }
 656
 657                 /* reset the engine and set to writable */
 658                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 659                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 660
 661                 /* load mc io regs */
 662                 for (i = 0; i < regs_size; i++) {
 663                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 664                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 665                 }
 666                 /* load the MC ucode */
 667                 fw_data = (const __be32 *)rdev->mc_fw->data;
 668                 for (i = 0; i < ucode_size; i++)
 669                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 670
 671                 /* put the engine back into the active state */
 672                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 673                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 674                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 675
 676                 /* wait for training to complete */
 677                 for (i = 0; i < rdev->usec_timeout; i++) {
 678                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 679                                 break;
 680                         udelay(1);
 681                 }
 682
 683                 if (running)
 684                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 685         }
 686
 687         return 0;
 688 }
 689
 690 int ni_init_microcode(struct radeon_device *rdev)
 691 {
 692         const char *chip_name;
 693         const char *rlc_chip_name;
 694         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 695         size_t smc_req_size = 0;
 696         char fw_name[30];
 697         int err;
 698
 699         DRM_DEBUG("\n");
 700
 701         switch (rdev->family) {
 702         case CHIP_BARTS:
 703                 chip_name = "BARTS";
 704                 rlc_chip_name = "BTC";
 705                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 706                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 707                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 708                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 709                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 710                 break;
 711         case CHIP_TURKS:
 712                 chip_name = "TURKS";
 713                 rlc_chip_name = "BTC";
 714                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 715                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 716                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 717                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 718                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 719                 break;
 720         case CHIP_CAICOS:
 721                 chip_name = "CAICOS";
 722                 rlc_chip_name = "BTC";
 723                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 724                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 725                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 726                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 727                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 728                 break;
 729         case CHIP_CAYMAN:
 730                 chip_name = "CAYMAN";
 731                 rlc_chip_name = "CAYMAN";
 732                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 733                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 734                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 735                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 736                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 737                 break;
 738         case CHIP_ARUBA:
 739                 chip_name = "ARUBA";
 740                 rlc_chip_name = "ARUBA";
 741                 /* pfp/me same size as CAYMAN */
 742                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 743                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 744                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 745                 mc_req_size = 0;
 746                 break;
 747         default: BUG();
 748         }
 749
 750         DRM_INFO("Loading %s Microcode\n", chip_name);
 751
 752         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 753         err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 754         if (err)
 755                 goto out;
 756         if (rdev->pfp_fw->size != pfp_req_size) {
 757                 printk(KERN_ERR
 758                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 759                        rdev->pfp_fw->size, fw_name);
 760                 err = -EINVAL;
 761                 goto out;
 762         }
 763
 764         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 765         err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
 766         if (err)
 767                 goto out;
 768         if (rdev->me_fw->size != me_req_size) {
 769                 printk(KERN_ERR
 770                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 771                        rdev->me_fw->size, fw_name);
 772                 err = -EINVAL;
 773         }
 774
 775         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", rlc_chip_name);
 776         err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 777         if (err)
 778                 goto out;
 779         if (rdev->rlc_fw->size != rlc_req_size) {
 780                 printk(KERN_ERR
 781                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 782                        rdev->rlc_fw->size, fw_name);
 783                 err = -EINVAL;
 784         }
 785
 786         /* no MC ucode on TN */
 787         if (!(rdev->flags & RADEON_IS_IGP)) {
 788                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 789                 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 790                 if (err)
 791                         goto out;
 792                 if (rdev->mc_fw->size != mc_req_size) {
 793                         printk(KERN_ERR
 794                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 795                                rdev->mc_fw->size, fw_name);
 796                         err = -EINVAL;
 797                 }
 798         }
 799
 800         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 801                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 802                 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 803                 if (err) {
 804                         printk(KERN_ERR
 805                                "smc: error loading firmware \"%s\"\n",
 806                                fw_name);
 807                         release_firmware(rdev->smc_fw);
 808                         rdev->smc_fw = NULL;
 809                         err = 0;
 810                 } else if (rdev->smc_fw->size != smc_req_size) {
 811                         printk(KERN_ERR
 812                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 813                                rdev->mc_fw->size, fw_name);
 814                         err = -EINVAL;
 815                 }
 816         }
 817
 818 out:
 819         if (err) {
 820                 if (err != -EINVAL)
 821                         printk(KERN_ERR
 822                                "ni_cp: Failed to load firmware \"%s\"\n",
 823                                fw_name);
 824                 release_firmware(rdev->pfp_fw);
 825                 rdev->pfp_fw = NULL;
 826                 release_firmware(rdev->me_fw);
 827                 rdev->me_fw = NULL;
 828                 release_firmware(rdev->rlc_fw);
 829                 rdev->rlc_fw = NULL;
 830                 release_firmware(rdev->mc_fw);
 831                 rdev->mc_fw = NULL;
 832         }
 833         return err;
 834 }
 835
 836 /**
 837  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 838  *
 839  * @rdev: radeon_device pointer
 840  * @reg: register offset in bytes
 841  * @val: register value
 842  *
 843  * Returns 0 for success or -EINVAL for an invalid register
 844  *
 845  */
 846 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 847                                      u32 reg, u32 *val)
 848 {
 849         switch (reg) {
 850         case GRBM_STATUS:
 851         case GRBM_STATUS_SE0:
 852         case GRBM_STATUS_SE1:
 853         case SRBM_STATUS:
 854         case SRBM_STATUS2:
 855         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 856         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 857         case UVD_STATUS:
 858                 *val = RREG32(reg);
 859                 return 0;
 860         default:
 861                 return -EINVAL;
 862         }
 863 }
 864
 865 int tn_get_temp(struct radeon_device *rdev)
 866 {
 867         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 868         int actual_temp = (temp / 8) - 49;
 869
 870         return actual_temp * 1000;
 871 }
 872
 873 /*
 874  * Core functions
 875  */
 876 static void cayman_gpu_init(struct radeon_device *rdev)
 877 {
 878         u32 gb_addr_config = 0;
 879         u32 mc_shared_chmap, mc_arb_ramcfg;
 880         u32 cgts_tcc_disable;
 881         u32 sx_debug_1;
 882         u32 smx_dc_ctl0;
 883         u32 cgts_sm_ctrl_reg;
 884         u32 hdp_host_path_cntl;
 885         u32 tmp;
 886         u32 disabled_rb_mask;
 887         int i, j;
 888
 889         switch (rdev->family) {
 890         case CHIP_CAYMAN:
 891                 rdev->config.cayman.max_shader_engines = 2;
 892                 rdev->config.cayman.max_pipes_per_simd = 4;
 893                 rdev->config.cayman.max_tile_pipes = 8;
 894                 rdev->config.cayman.max_simds_per_se = 12;
 895                 rdev->config.cayman.max_backends_per_se = 4;
 896                 rdev->config.cayman.max_texture_channel_caches = 8;
 897                 rdev->config.cayman.max_gprs = 256;
 898                 rdev->config.cayman.max_threads = 256;
 899                 rdev->config.cayman.max_gs_threads = 32;
 900                 rdev->config.cayman.max_stack_entries = 512;
 901                 rdev->config.cayman.sx_num_of_sets = 8;
 902                 rdev->config.cayman.sx_max_export_size = 256;
 903                 rdev->config.cayman.sx_max_export_pos_size = 64;
 904                 rdev->config.cayman.sx_max_export_smx_size = 192;
 905                 rdev->config.cayman.max_hw_contexts = 8;
 906                 rdev->config.cayman.sq_num_cf_insts = 2;
 907
 908                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 909                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 910                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 911                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 912                 break;
 913         case CHIP_ARUBA:
 914         default:
 915                 rdev->config.cayman.max_shader_engines = 1;
 916                 rdev->config.cayman.max_pipes_per_simd = 4;
 917                 rdev->config.cayman.max_tile_pipes = 2;
 918                 if ((rdev->pdev->device == 0x9900) ||
 919                     (rdev->pdev->device == 0x9901) ||
 920                     (rdev->pdev->device == 0x9905) ||
 921                     (rdev->pdev->device == 0x9906) ||
 922                     (rdev->pdev->device == 0x9907) ||
 923                     (rdev->pdev->device == 0x9908) ||
 924                     (rdev->pdev->device == 0x9909) ||
 925                     (rdev->pdev->device == 0x990B) ||
 926                     (rdev->pdev->device == 0x990C) ||
 927                     (rdev->pdev->device == 0x990F) ||
 928                     (rdev->pdev->device == 0x9910) ||
 929                     (rdev->pdev->device == 0x9917) ||
 930                     (rdev->pdev->device == 0x9999) ||
 931                     (rdev->pdev->device == 0x999C)) {
 932                         rdev->config.cayman.max_simds_per_se = 6;
 933                         rdev->config.cayman.max_backends_per_se = 2;
 934                         rdev->config.cayman.max_hw_contexts = 8;
 935                         rdev->config.cayman.sx_max_export_size = 256;
 936                         rdev->config.cayman.sx_max_export_pos_size = 64;
 937                         rdev->config.cayman.sx_max_export_smx_size = 192;
 938                 } else if ((rdev->pdev->device == 0x9903) ||
 939                            (rdev->pdev->device == 0x9904) ||
 940                            (rdev->pdev->device == 0x990A) ||
 941                            (rdev->pdev->device == 0x990D) ||
 942                            (rdev->pdev->device == 0x990E) ||
 943                            (rdev->pdev->device == 0x9913) ||
 944                            (rdev->pdev->device == 0x9918) ||
 945                            (rdev->pdev->device == 0x999D)) {
 946                         rdev->config.cayman.max_simds_per_se = 4;
 947                         rdev->config.cayman.max_backends_per_se = 2;
 948                         rdev->config.cayman.max_hw_contexts = 8;
 949                         rdev->config.cayman.sx_max_export_size = 256;
 950                         rdev->config.cayman.sx_max_export_pos_size = 64;
 951                         rdev->config.cayman.sx_max_export_smx_size = 192;
 952                 } else if ((rdev->pdev->device == 0x9919) ||
 953                            (rdev->pdev->device == 0x9990) ||
 954                            (rdev->pdev->device == 0x9991) ||
 955                            (rdev->pdev->device == 0x9994) ||
 956                            (rdev->pdev->device == 0x9995) ||
 957                            (rdev->pdev->device == 0x9996) ||
 958                            (rdev->pdev->device == 0x999A) ||
 959                            (rdev->pdev->device == 0x99A0)) {
 960                         rdev->config.cayman.max_simds_per_se = 3;
 961                         rdev->config.cayman.max_backends_per_se = 1;
 962                         rdev->config.cayman.max_hw_contexts = 4;
 963                         rdev->config.cayman.sx_max_export_size = 128;
 964                         rdev->config.cayman.sx_max_export_pos_size = 32;
 965                         rdev->config.cayman.sx_max_export_smx_size = 96;
 966                 } else {
 967                         rdev->config.cayman.max_simds_per_se = 2;
 968                         rdev->config.cayman.max_backends_per_se = 1;
 969                         rdev->config.cayman.max_hw_contexts = 4;
 970                         rdev->config.cayman.sx_max_export_size = 128;
 971                         rdev->config.cayman.sx_max_export_pos_size = 32;
 972                         rdev->config.cayman.sx_max_export_smx_size = 96;
 973                 }
 974                 rdev->config.cayman.max_texture_channel_caches = 2;
 975                 rdev->config.cayman.max_gprs = 256;
 976                 rdev->config.cayman.max_threads = 256;
 977                 rdev->config.cayman.max_gs_threads = 32;
 978                 rdev->config.cayman.max_stack_entries = 512;
 979                 rdev->config.cayman.sx_num_of_sets = 8;
 980                 rdev->config.cayman.sq_num_cf_insts = 2;
 981
 982                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 983                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 984                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 985                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 986                 break;
 987         }
 988
 989         /* Initialize HDP */
 990         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 991                 WREG32((0x2c14 + j), 0x00000000);
 992                 WREG32((0x2c18 + j), 0x00000000);
 993                 WREG32((0x2c1c + j), 0x00000000);
 994                 WREG32((0x2c20 + j), 0x00000000);
 995                 WREG32((0x2c24 + j), 0x00000000);
 996         }
 997
 998         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 999         WREG32(SRBM_INT_CNTL, 0x1);
1000         WREG32(SRBM_INT_ACK, 0x1);
1001
1002         evergreen_fix_pci_max_read_req_size(rdev);
1003
1004         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1005         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1006
1007         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1008         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1009         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1010                 rdev->config.cayman.mem_row_size_in_kb = 4;
1011         /* XXX use MC settings? */
1012         rdev->config.cayman.shader_engine_tile_size = 32;
1013         rdev->config.cayman.num_gpus = 1;
1014         rdev->config.cayman.multi_gpu_tile_size = 64;
1015
1016         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1017         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1018         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1019         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1020         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1021         rdev->config.cayman.num_shader_engines = tmp + 1;
1022         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1023         rdev->config.cayman.num_gpus = tmp + 1;
1024         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1025         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1026         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1027         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1028
1029
1030         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1031          * not have bank info, so create a custom tiling dword.
1032          * bits 3:0   num_pipes
1033          * bits 7:4   num_banks
1034          * bits 11:8  group_size
1035          * bits 15:12 row_size
1036          */
1037         rdev->config.cayman.tile_config = 0;
1038         switch (rdev->config.cayman.num_tile_pipes) {
1039         case 1:
1040         default:
1041                 rdev->config.cayman.tile_config |= (0 << 0);
1042                 break;
1043         case 2:
1044                 rdev->config.cayman.tile_config |= (1 << 0);
1045                 break;
1046         case 4:
1047                 rdev->config.cayman.tile_config |= (2 << 0);
1048                 break;
1049         case 8:
1050                 rdev->config.cayman.tile_config |= (3 << 0);
1051                 break;
1052         }
1053
1054         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1055         if (rdev->flags & RADEON_IS_IGP)
1056                 rdev->config.cayman.tile_config |= 1 << 4;
1057         else {
1058                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1059                 case 0: /* four banks */
1060                         rdev->config.cayman.tile_config |= 0 << 4;
1061                         break;
1062                 case 1: /* eight banks */
1063                         rdev->config.cayman.tile_config |= 1 << 4;
1064                         break;
1065                 case 2: /* sixteen banks */
1066                 default:
1067                         rdev->config.cayman.tile_config |= 2 << 4;
1068                         break;
1069                 }
1070         }
1071         rdev->config.cayman.tile_config |=
1072                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1073         rdev->config.cayman.tile_config |=
1074                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1075
1076         tmp = 0;
1077         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1078                 u32 rb_disable_bitmap;
1079
1080                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1081                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1082                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1083                 tmp <<= 4;
1084                 tmp |= rb_disable_bitmap;
1085         }
1086         /* enabled rb are just the one not disabled :) */
1087         disabled_rb_mask = tmp;
1088         tmp = 0;
1089         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1090                 tmp |= (1 << i);
1091         /* if all the backends are disabled, fix it up here */
1092         if ((disabled_rb_mask & tmp) == tmp) {
1093                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1094                         disabled_rb_mask &= ~(1 << i);
1095         }
1096
1097         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1098                 u32 simd_disable_bitmap;
1099
1100                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1101                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1102                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1103                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1104                 tmp <<= 16;
1105                 tmp |= simd_disable_bitmap;
1106         }
1107         rdev->config.cayman.active_simds = hweight32(~tmp);
1108
1109         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1110         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1111
1112         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1113         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1114         if (ASIC_IS_DCE6(rdev))
1115                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1116         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1117         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1118         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1119         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1120         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1121         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1122
1123         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1124             (rdev->flags & RADEON_IS_IGP)) {
1125                 if ((disabled_rb_mask & 3) == 2) {
1126                         /* RB1 disabled, RB0 enabled */
1127                         tmp = 0x00000000;
1128                 } else {
1129                         /* RB0 disabled, RB1 enabled */
1130                         tmp = 0x11111111;
1131                 }
1132         } else {
1133                 tmp = gb_addr_config & NUM_PIPES_MASK;
1134                 tmp = r6xx_remap_render_backend(rdev, tmp,
1135                                                 rdev->config.cayman.max_backends_per_se *
1136                                                 rdev->config.cayman.max_shader_engines,
1137                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1138         }
1139         WREG32(GB_BACKEND_MAP, tmp);
1140
1141         cgts_tcc_disable = 0xffff0000;
1142         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1143                 cgts_tcc_disable &= ~(1 << (16 + i));
1144         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1145         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1146         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1147         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1148
1149         /* reprogram the shader complex */
1150         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1151         for (i = 0; i < 16; i++)
1152                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1153         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1154
1155         /* set HW defaults for 3D engine */
1156         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1157
1158         sx_debug_1 = RREG32(SX_DEBUG_1);
1159         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1160         WREG32(SX_DEBUG_1, sx_debug_1);
1161
1162         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1163         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1164         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1165         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1166
1167         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1168
1169         /* need to be explicitly zero-ed */
1170         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1171         WREG32(SQ_LSTMP_RING_BASE, 0);
1172         WREG32(SQ_HSTMP_RING_BASE, 0);
1173         WREG32(SQ_ESTMP_RING_BASE, 0);
1174         WREG32(SQ_GSTMP_RING_BASE, 0);
1175         WREG32(SQ_VSTMP_RING_BASE, 0);
1176         WREG32(SQ_PSTMP_RING_BASE, 0);
1177
1178         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1179
1180         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1181                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1182                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1183
1184         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1185                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1186                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1187
1188
1189         WREG32(VGT_NUM_INSTANCES, 1);
1190
1191         WREG32(CP_PERFMON_CNTL, 0);
1192
1193         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1194                                   FETCH_FIFO_HIWATER(0x4) |
1195                                   DONE_FIFO_HIWATER(0xe0) |
1196                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1197
1198         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1199         WREG32(SQ_CONFIG, (VC_ENABLE |
1200                            EXPORT_SRC_C |
1201                            GFX_PRIO(0) |
1202                            CS1_PRIO(0) |
1203                            CS2_PRIO(1)));
1204         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1205
1206         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1207                                           FORCE_EOV_MAX_REZ_CNT(255)));
1208
1209         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1210                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1211
1212         WREG32(VGT_GS_VERTEX_REUSE, 16);
1213         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1214
1215         WREG32(CB_PERF_CTR0_SEL_0, 0);
1216         WREG32(CB_PERF_CTR0_SEL_1, 0);
1217         WREG32(CB_PERF_CTR1_SEL_0, 0);
1218         WREG32(CB_PERF_CTR1_SEL_1, 0);
1219         WREG32(CB_PERF_CTR2_SEL_0, 0);
1220         WREG32(CB_PERF_CTR2_SEL_1, 0);
1221         WREG32(CB_PERF_CTR3_SEL_0, 0);
1222         WREG32(CB_PERF_CTR3_SEL_1, 0);
1223
1224         tmp = RREG32(HDP_MISC_CNTL);
1225         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1226         WREG32(HDP_MISC_CNTL, tmp);
1227
1228         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1229         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1230
1231         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1232
1233         udelay(50);
1234
1235         /* set clockgating golden values on TN */
1236         if (rdev->family == CHIP_ARUBA) {
1237                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1238                 tmp &= ~0x00380000;
1239                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1240                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1241                 tmp &= ~0x0e000000;
1242                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1243         }
1244 }
1245
1246 /*
1247  * GART
1248  */
1249 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1250 {
1251         /* flush hdp cache */
1252         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1253
1254         /* bits 0-7 are the VM contexts0-7 */
1255         WREG32(VM_INVALIDATE_REQUEST, 1);
1256 }
1257
1258 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1259 {
1260         int i, r;
1261
1262         if (rdev->gart.robj == NULL) {
1263                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1264                 return -EINVAL;
1265         }
1266         r = radeon_gart_table_vram_pin(rdev);
1267         if (r)
1268                 return r;
1269         /* Setup TLB control */
1270         WREG32(MC_VM_MX_L1_TLB_CNTL,
1271                (0xA << 7) |
1272                ENABLE_L1_TLB |
1273                ENABLE_L1_FRAGMENT_PROCESSING |
1274                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1275                ENABLE_ADVANCED_DRIVER_MODEL |
1276                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1277         /* Setup L2 cache */
1278         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1279                ENABLE_L2_FRAGMENT_PROCESSING |
1280                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1281                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1282                EFFECTIVE_L2_QUEUE_SIZE(7) |
1283                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1284         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1285         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1286                BANK_SELECT(6) |
1287                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1288         /* setup context0 */
1289         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1290         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1291         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1292         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1293                         (u32)(rdev->dummy_page.addr >> 12));
1294         WREG32(VM_CONTEXT0_CNTL2, 0);
1295         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1296                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1297
1298         WREG32(0x15D4, 0);
1299         WREG32(0x15D8, 0);
1300         WREG32(0x15DC, 0);
1301
1302         /* empty context1-7 */
1303         /* Assign the pt base to something valid for now; the pts used for
1304          * the VMs are determined by the application and setup and assigned
1305          * on the fly in the vm part of radeon_gart.c
1306          */
1307         for (i = 1; i < 8; i++) {
1308                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1309                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1310                         rdev->vm_manager.max_pfn - 1);
1311                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1312                        rdev->vm_manager.saved_table_addr[i]);
1313         }
1314
1315         /* enable context1-7 */
1316         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1317                (u32)(rdev->dummy_page.addr >> 12));
1318         WREG32(VM_CONTEXT1_CNTL2, 4);
1319         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1320                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1321                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1322                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1323                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1324                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1325                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1326                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1327                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1328                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1329                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1330                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1331                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1332                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1333
1334         cayman_pcie_gart_tlb_flush(rdev);
1335         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1336                  (unsigned)(rdev->mc.gtt_size >> 20),
1337                  (unsigned long long)rdev->gart.table_addr);
1338         rdev->gart.ready = true;
1339         return 0;
1340 }
1341
1342 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1343 {
1344         unsigned i;
1345
1346         for (i = 1; i < 8; ++i) {
1347                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1348                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1349         }
1350
1351         /* Disable all tables */
1352         WREG32(VM_CONTEXT0_CNTL, 0);
1353         WREG32(VM_CONTEXT1_CNTL, 0);
1354         /* Setup TLB control */
1355         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1356                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1357                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1358         /* Setup L2 cache */
1359         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1360                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1361                EFFECTIVE_L2_QUEUE_SIZE(7) |
1362                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1363         WREG32(VM_L2_CNTL2, 0);
1364         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1365                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1366         radeon_gart_table_vram_unpin(rdev);
1367 }
1368
1369 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1370 {
1371         cayman_pcie_gart_disable(rdev);
1372         radeon_gart_table_vram_free(rdev);
1373         radeon_gart_fini(rdev);
1374 }
1375
1376 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1377                               int ring, u32 cp_int_cntl)
1378 {
1379         WREG32(SRBM_GFX_CNTL, RINGID(ring));
1380         WREG32(CP_INT_CNTL, cp_int_cntl);
1381 }
1382
1383 /*
1384  * CP.
1385  */
1386 void cayman_fence_ring_emit(struct radeon_device *rdev,
1387                             struct radeon_fence *fence)
1388 {
1389         struct radeon_ring *ring = &rdev->ring[fence->ring];
1390         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1391         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1392                 PACKET3_SH_ACTION_ENA;
1393
1394         /* flush read cache over gart for this vmid */
1395         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1396         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1397         radeon_ring_write(ring, 0xFFFFFFFF);
1398         radeon_ring_write(ring, 0);
1399         radeon_ring_write(ring, 10); /* poll interval */
1400         /* EVENT_WRITE_EOP - flush caches, send int */
1401         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1402         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1403         radeon_ring_write(ring, lower_32_bits(addr));
1404         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1405         radeon_ring_write(ring, fence->seq);
1406         radeon_ring_write(ring, 0);
1407 }
1408
1409 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1410 {
1411         struct radeon_ring *ring = &rdev->ring[ib->ring];
1412         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1413         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1414                 PACKET3_SH_ACTION_ENA;
1415
1416         /* set to DX10/11 mode */
1417         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1418         radeon_ring_write(ring, 1);
1419
1420         if (ring->rptr_save_reg) {
1421                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1422                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1423                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1424                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1425                 radeon_ring_write(ring, next_rptr);
1426         }
1427
1428         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1429         radeon_ring_write(ring,
1430 #ifdef __BIG_ENDIAN
1431                           (2 << 0) |
1432 #endif
1433                           (ib->gpu_addr & 0xFFFFFFFC));
1434         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1435         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1436
1437         /* flush read cache over gart for this vmid */
1438         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1439         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1440         radeon_ring_write(ring, 0xFFFFFFFF);
1441         radeon_ring_write(ring, 0);
1442         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1443 }
1444
1445 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1446 {
1447         if (enable)
1448                 WREG32(CP_ME_CNTL, 0);
1449         else {
1450                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1451                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1452                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1453                 WREG32(SCRATCH_UMSK, 0);
1454                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1455         }
1456 }
1457
1458 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1459                         struct radeon_ring *ring)
1460 {
1461         u32 rptr;
1462
1463         if (rdev->wb.enabled)
1464                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1465         else {
1466                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1467                         rptr = RREG32(CP_RB0_RPTR);
1468                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1469                         rptr = RREG32(CP_RB1_RPTR);
1470                 else
1471                         rptr = RREG32(CP_RB2_RPTR);
1472         }
1473
1474         return rptr;
1475 }
1476
1477 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1478                         struct radeon_ring *ring)
1479 {
1480         u32 wptr;
1481
1482         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1483                 wptr = RREG32(CP_RB0_WPTR);
1484         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1485                 wptr = RREG32(CP_RB1_WPTR);
1486         else
1487                 wptr = RREG32(CP_RB2_WPTR);
1488
1489         return wptr;
1490 }
1491
1492 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1493                          struct radeon_ring *ring)
1494 {
1495         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1496                 WREG32(CP_RB0_WPTR, ring->wptr);
1497                 (void)RREG32(CP_RB0_WPTR);
1498         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1499                 WREG32(CP_RB1_WPTR, ring->wptr);
1500                 (void)RREG32(CP_RB1_WPTR);
1501         } else {
1502                 WREG32(CP_RB2_WPTR, ring->wptr);
1503                 (void)RREG32(CP_RB2_WPTR);
1504         }
1505 }
1506
1507 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1508 {
1509         const __be32 *fw_data;
1510         int i;
1511
1512         if (!rdev->me_fw || !rdev->pfp_fw)
1513                 return -EINVAL;
1514
1515         cayman_cp_enable(rdev, false);
1516
1517         fw_data = (const __be32 *)rdev->pfp_fw->data;
1518         WREG32(CP_PFP_UCODE_ADDR, 0);
1519         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1520                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1521         WREG32(CP_PFP_UCODE_ADDR, 0);
1522
1523         fw_data = (const __be32 *)rdev->me_fw->data;
1524         WREG32(CP_ME_RAM_WADDR, 0);
1525         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1526                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1527
1528         WREG32(CP_PFP_UCODE_ADDR, 0);
1529         WREG32(CP_ME_RAM_WADDR, 0);
1530         WREG32(CP_ME_RAM_RADDR, 0);
1531         return 0;
1532 }
1533
1534 static int cayman_cp_start(struct radeon_device *rdev)
1535 {
1536         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1537         int r, i;
1538
1539         r = radeon_ring_lock(rdev, ring, 7);
1540         if (r) {
1541                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1542                 return r;
1543         }
1544         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1545         radeon_ring_write(ring, 0x1);
1546         radeon_ring_write(ring, 0x0);
1547         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1548         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1549         radeon_ring_write(ring, 0);
1550         radeon_ring_write(ring, 0);
1551         radeon_ring_unlock_commit(rdev, ring, false);
1552
1553         cayman_cp_enable(rdev, true);
1554
1555         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1556         if (r) {
1557                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1558                 return r;
1559         }
1560
1561         /* setup clear context state */
1562         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1563         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1564
1565         for (i = 0; i < cayman_default_size; i++)
1566                 radeon_ring_write(ring, cayman_default_state[i]);
1567
1568         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1569         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1570
1571         /* set clear context state */
1572         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1573         radeon_ring_write(ring, 0);
1574
1575         /* SQ_VTX_BASE_VTX_LOC */
1576         radeon_ring_write(ring, 0xc0026f00);
1577         radeon_ring_write(ring, 0x00000000);
1578         radeon_ring_write(ring, 0x00000000);
1579         radeon_ring_write(ring, 0x00000000);
1580
1581         /* Clear consts */
1582         radeon_ring_write(ring, 0xc0036f00);
1583         radeon_ring_write(ring, 0x00000bc4);
1584         radeon_ring_write(ring, 0xffffffff);
1585         radeon_ring_write(ring, 0xffffffff);
1586         radeon_ring_write(ring, 0xffffffff);
1587
1588         radeon_ring_write(ring, 0xc0026900);
1589         radeon_ring_write(ring, 0x00000316);
1590         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1591         radeon_ring_write(ring, 0x00000010); /*  */
1592
1593         radeon_ring_unlock_commit(rdev, ring, false);
1594
1595         /* XXX init other rings */
1596
1597         return 0;
1598 }
1599
1600 static void cayman_cp_fini(struct radeon_device *rdev)
1601 {
1602         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1603         cayman_cp_enable(rdev, false);
1604         radeon_ring_fini(rdev, ring);
1605         radeon_scratch_free(rdev, ring->rptr_save_reg);
1606 }
1607
1608 static int cayman_cp_resume(struct radeon_device *rdev)
1609 {
1610         static const int ridx[] = {
1611                 RADEON_RING_TYPE_GFX_INDEX,
1612                 CAYMAN_RING_TYPE_CP1_INDEX,
1613                 CAYMAN_RING_TYPE_CP2_INDEX
1614         };
1615         static const unsigned cp_rb_cntl[] = {
1616                 CP_RB0_CNTL,
1617                 CP_RB1_CNTL,
1618                 CP_RB2_CNTL,
1619         };
1620         static const unsigned cp_rb_rptr_addr[] = {
1621                 CP_RB0_RPTR_ADDR,
1622                 CP_RB1_RPTR_ADDR,
1623                 CP_RB2_RPTR_ADDR
1624         };
1625         static const unsigned cp_rb_rptr_addr_hi[] = {
1626                 CP_RB0_RPTR_ADDR_HI,
1627                 CP_RB1_RPTR_ADDR_HI,
1628                 CP_RB2_RPTR_ADDR_HI
1629         };
1630         static const unsigned cp_rb_base[] = {
1631                 CP_RB0_BASE,
1632                 CP_RB1_BASE,
1633                 CP_RB2_BASE
1634         };
1635         static const unsigned cp_rb_rptr[] = {
1636                 CP_RB0_RPTR,
1637                 CP_RB1_RPTR,
1638                 CP_RB2_RPTR
1639         };
1640         static const unsigned cp_rb_wptr[] = {
1641                 CP_RB0_WPTR,
1642                 CP_RB1_WPTR,
1643                 CP_RB2_WPTR
1644         };
1645         struct radeon_ring *ring;
1646         int i, r;
1647
1648         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1649         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1650                                  SOFT_RESET_PA |
1651                                  SOFT_RESET_SH |
1652                                  SOFT_RESET_VGT |
1653                                  SOFT_RESET_SPI |
1654                                  SOFT_RESET_SX));
1655         RREG32(GRBM_SOFT_RESET);
1656         mdelay(15);
1657         WREG32(GRBM_SOFT_RESET, 0);
1658         RREG32(GRBM_SOFT_RESET);
1659
1660         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1661         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1662
1663         /* Set the write pointer delay */
1664         WREG32(CP_RB_WPTR_DELAY, 0);
1665
1666         WREG32(CP_DEBUG, (1 << 27));
1667
1668         /* set the wb address whether it's enabled or not */
1669         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1670         WREG32(SCRATCH_UMSK, 0xff);
1671
1672         for (i = 0; i < 3; ++i) {
1673                 uint32_t rb_cntl;
1674                 uint64_t addr;
1675
1676                 /* Set ring buffer size */
1677                 ring = &rdev->ring[ridx[i]];
1678                 rb_cntl = order_base_2(ring->ring_size / 8);
1679                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1680 #ifdef __BIG_ENDIAN
1681                 rb_cntl |= BUF_SWAP_32BIT;
1682 #endif
1683                 WREG32(cp_rb_cntl[i], rb_cntl);
1684
1685                 /* set the wb address whether it's enabled or not */
1686                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1687                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1688                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1689         }
1690
1691         /* set the rb base addr, this causes an internal reset of ALL rings */
1692         for (i = 0; i < 3; ++i) {
1693                 ring = &rdev->ring[ridx[i]];
1694                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1695         }
1696
1697         for (i = 0; i < 3; ++i) {
1698                 /* Initialize the ring buffer's read and write pointers */
1699                 ring = &rdev->ring[ridx[i]];
1700                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1701
1702                 ring->wptr = 0;
1703                 WREG32(cp_rb_rptr[i], 0);
1704                 WREG32(cp_rb_wptr[i], ring->wptr);
1705
1706                 mdelay(1);
1707                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1708         }
1709
1710         /* start the rings */
1711         cayman_cp_start(rdev);
1712         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1713         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1714         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1715         /* this only test cp0 */
1716         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1717         if (r) {
1718                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1719                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1720                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1721                 return r;
1722         }
1723
1724         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1725                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1726
1727         return 0;
1728 }
1729
1730 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1731 {
1732         u32 reset_mask = 0;
1733         u32 tmp;
1734
1735         /* GRBM_STATUS */
1736         tmp = RREG32(GRBM_STATUS);
1737         if (tmp & (PA_BUSY | SC_BUSY |
1738                    SH_BUSY | SX_BUSY |
1739                    TA_BUSY | VGT_BUSY |
1740                    DB_BUSY | CB_BUSY |
1741                    GDS_BUSY | SPI_BUSY |
1742                    IA_BUSY | IA_BUSY_NO_DMA))
1743                 reset_mask |= RADEON_RESET_GFX;
1744
1745         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1746                    CP_BUSY | CP_COHERENCY_BUSY))
1747                 reset_mask |= RADEON_RESET_CP;
1748
1749         if (tmp & GRBM_EE_BUSY)
1750                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1751
1752         /* DMA_STATUS_REG 0 */
1753         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1754         if (!(tmp & DMA_IDLE))
1755                 reset_mask |= RADEON_RESET_DMA;
1756
1757         /* DMA_STATUS_REG 1 */
1758         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1759         if (!(tmp & DMA_IDLE))
1760                 reset_mask |= RADEON_RESET_DMA1;
1761
1762         /* SRBM_STATUS2 */
1763         tmp = RREG32(SRBM_STATUS2);
1764         if (tmp & DMA_BUSY)
1765                 reset_mask |= RADEON_RESET_DMA;
1766
1767         if (tmp & DMA1_BUSY)
1768                 reset_mask |= RADEON_RESET_DMA1;
1769
1770         /* SRBM_STATUS */
1771         tmp = RREG32(SRBM_STATUS);
1772         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1773                 reset_mask |= RADEON_RESET_RLC;
1774
1775         if (tmp & IH_BUSY)
1776                 reset_mask |= RADEON_RESET_IH;
1777
1778         if (tmp & SEM_BUSY)
1779                 reset_mask |= RADEON_RESET_SEM;
1780
1781         if (tmp & GRBM_RQ_PENDING)
1782                 reset_mask |= RADEON_RESET_GRBM;
1783
1784         if (tmp & VMC_BUSY)
1785                 reset_mask |= RADEON_RESET_VMC;
1786
1787         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1788                    MCC_BUSY | MCD_BUSY))
1789                 reset_mask |= RADEON_RESET_MC;
1790
1791         if (evergreen_is_display_hung(rdev))
1792                 reset_mask |= RADEON_RESET_DISPLAY;
1793
1794         /* VM_L2_STATUS */
1795         tmp = RREG32(VM_L2_STATUS);
1796         if (tmp & L2_BUSY)
1797                 reset_mask |= RADEON_RESET_VMC;
1798
1799         /* Skip MC reset as it's mostly likely not hung, just busy */
1800         if (reset_mask & RADEON_RESET_MC) {
1801                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1802                 reset_mask &= ~RADEON_RESET_MC;
1803         }
1804
1805         return reset_mask;
1806 }
1807
1808 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1809 {
1810         struct evergreen_mc_save save;
1811         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1812         u32 tmp;
1813
1814         if (reset_mask == 0)
1815                 return;
1816
1817         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1818
1819         evergreen_print_gpu_status_regs(rdev);
1820         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1821                  RREG32(0x14F8));
1822         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1823                  RREG32(0x14D8));
1824         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1825                  RREG32(0x14FC));
1826         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1827                  RREG32(0x14DC));
1828
1829         /* Disable CP parsing/prefetching */
1830         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1831
1832         if (reset_mask & RADEON_RESET_DMA) {
1833                 /* dma0 */
1834                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1835                 tmp &= ~DMA_RB_ENABLE;
1836                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1837         }
1838
1839         if (reset_mask & RADEON_RESET_DMA1) {
1840                 /* dma1 */
1841                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1842                 tmp &= ~DMA_RB_ENABLE;
1843                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1844         }
1845
1846         udelay(50);
1847
1848         evergreen_mc_stop(rdev, &save);
1849         if (evergreen_mc_wait_for_idle(rdev)) {
1850                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1851         }
1852
1853         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1854                 grbm_soft_reset = SOFT_RESET_CB |
1855                         SOFT_RESET_DB |
1856                         SOFT_RESET_GDS |
1857                         SOFT_RESET_PA |
1858                         SOFT_RESET_SC |
1859                         SOFT_RESET_SPI |
1860                         SOFT_RESET_SH |
1861                         SOFT_RESET_SX |
1862                         SOFT_RESET_TC |
1863                         SOFT_RESET_TA |
1864                         SOFT_RESET_VGT |
1865                         SOFT_RESET_IA;
1866         }
1867
1868         if (reset_mask & RADEON_RESET_CP) {
1869                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1870
1871                 srbm_soft_reset |= SOFT_RESET_GRBM;
1872         }
1873
1874         if (reset_mask & RADEON_RESET_DMA)
1875                 srbm_soft_reset |= SOFT_RESET_DMA;
1876
1877         if (reset_mask & RADEON_RESET_DMA1)
1878                 srbm_soft_reset |= SOFT_RESET_DMA1;
1879
1880         if (reset_mask & RADEON_RESET_DISPLAY)
1881                 srbm_soft_reset |= SOFT_RESET_DC;
1882
1883         if (reset_mask & RADEON_RESET_RLC)
1884                 srbm_soft_reset |= SOFT_RESET_RLC;
1885
1886         if (reset_mask & RADEON_RESET_SEM)
1887                 srbm_soft_reset |= SOFT_RESET_SEM;
1888
1889         if (reset_mask & RADEON_RESET_IH)
1890                 srbm_soft_reset |= SOFT_RESET_IH;
1891
1892         if (reset_mask & RADEON_RESET_GRBM)
1893                 srbm_soft_reset |= SOFT_RESET_GRBM;
1894
1895         if (reset_mask & RADEON_RESET_VMC)
1896                 srbm_soft_reset |= SOFT_RESET_VMC;
1897
1898         if (!(rdev->flags & RADEON_IS_IGP)) {
1899                 if (reset_mask & RADEON_RESET_MC)
1900                         srbm_soft_reset |= SOFT_RESET_MC;
1901         }
1902
1903         if (grbm_soft_reset) {
1904                 tmp = RREG32(GRBM_SOFT_RESET);
1905                 tmp |= grbm_soft_reset;
1906                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1907                 WREG32(GRBM_SOFT_RESET, tmp);
1908                 tmp = RREG32(GRBM_SOFT_RESET);
1909
1910                 udelay(50);
1911
1912                 tmp &= ~grbm_soft_reset;
1913                 WREG32(GRBM_SOFT_RESET, tmp);
1914                 tmp = RREG32(GRBM_SOFT_RESET);
1915         }
1916
1917         if (srbm_soft_reset) {
1918                 tmp = RREG32(SRBM_SOFT_RESET);
1919                 tmp |= srbm_soft_reset;
1920                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1921                 WREG32(SRBM_SOFT_RESET, tmp);
1922                 tmp = RREG32(SRBM_SOFT_RESET);
1923
1924                 udelay(50);
1925
1926                 tmp &= ~srbm_soft_reset;
1927                 WREG32(SRBM_SOFT_RESET, tmp);
1928                 tmp = RREG32(SRBM_SOFT_RESET);
1929         }
1930
1931         /* Wait a little for things to settle down */
1932         udelay(50);
1933
1934         evergreen_mc_resume(rdev, &save);
1935         udelay(50);
1936
1937         evergreen_print_gpu_status_regs(rdev);
1938 }
1939
1940 int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1941 {
1942         u32 reset_mask;
1943
1944         if (hard) {
1945                 evergreen_gpu_pci_config_reset(rdev);
1946                 return 0;
1947         }
1948
1949         reset_mask = cayman_gpu_check_soft_reset(rdev);
1950
1951         if (reset_mask)
1952                 r600_set_bios_scratch_engine_hung(rdev, true);
1953
1954         cayman_gpu_soft_reset(rdev, reset_mask);
1955
1956         reset_mask = cayman_gpu_check_soft_reset(rdev);
1957
1958         if (reset_mask)
1959                 evergreen_gpu_pci_config_reset(rdev);
1960
1961         r600_set_bios_scratch_engine_hung(rdev, false);
1962
1963         return 0;
1964 }
1965
1966 /**
1967  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1968  *
1969  * @rdev: radeon_device pointer
1970  * @ring: radeon_ring structure holding ring information
1971  *
1972  * Check if the GFX engine is locked up.
1973  * Returns true if the engine appears to be locked up, false if not.
1974  */
1975 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1976 {
1977         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1978
1979         if (!(reset_mask & (RADEON_RESET_GFX |
1980                             RADEON_RESET_COMPUTE |
1981                             RADEON_RESET_CP))) {
1982                 radeon_ring_lockup_update(rdev, ring);
1983                 return false;
1984         }
1985         return radeon_ring_test_lockup(rdev, ring);
1986 }
1987
1988 static void cayman_uvd_init(struct radeon_device *rdev)
1989 {
1990         int r;
1991
1992         if (!rdev->has_uvd)
1993                 return;
1994
1995         r = radeon_uvd_init(rdev);
1996         if (r) {
1997                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1998                 /*
1999                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
2000                  * to early fails uvd_v2_2_resume() and thus nothing happens
2001                  * there. So it is pointless to try to go through that code
2002                  * hence why we disable uvd here.
2003                  */
2004                 rdev->has_uvd = 0;
2005                 return;
2006         }
2007         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
2008         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
2009 }
2010
2011 static void cayman_uvd_start(struct radeon_device *rdev)
2012 {
2013         int r;
2014
2015         if (!rdev->has_uvd)
2016                 return;
2017
2018         r = uvd_v2_2_resume(rdev);
2019         if (r) {
2020                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2021                 goto error;
2022         }
2023         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2024         if (r) {
2025                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2026                 goto error;
2027         }
2028         return;
2029
2030 error:
2031         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2032 }
2033
2034 static void cayman_uvd_resume(struct radeon_device *rdev)
2035 {
2036         struct radeon_ring *ring;
2037         int r;
2038
2039         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2040                 return;
2041
2042         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2043         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2044         if (r) {
2045                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2046                 return;
2047         }
2048         r = uvd_v1_0_init(rdev);
2049         if (r) {
2050                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2051                 return;
2052         }
2053 }
2054
2055 static void cayman_vce_init(struct radeon_device *rdev)
2056 {
2057         int r;
2058
2059         /* Only set for CHIP_ARUBA */
2060         if (!rdev->has_vce)
2061                 return;
2062
2063         r = radeon_vce_init(rdev);
2064         if (r) {
2065                 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2066                 /*
2067                  * At this point rdev->vce.vcpu_bo is NULL which trickles down
2068                  * to early fails cayman_vce_start() and thus nothing happens
2069                  * there. So it is pointless to try to go through that code
2070                  * hence why we disable vce here.
2071                  */
2072                 rdev->has_vce = 0;
2073                 return;
2074         }
2075         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2076         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2077         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2078         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2079 }
2080
2081 static void cayman_vce_start(struct radeon_device *rdev)
2082 {
2083         int r;
2084
2085         if (!rdev->has_vce)
2086                 return;
2087
2088         r = radeon_vce_resume(rdev);
2089         if (r) {
2090                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2091                 goto error;
2092         }
2093         r = vce_v1_0_resume(rdev);
2094         if (r) {
2095                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2096                 goto error;
2097         }
2098         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2099         if (r) {
2100                 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2101                 goto error;
2102         }
2103         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2104         if (r) {
2105                 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2106                 goto error;
2107         }
2108         return;
2109
2110 error:
2111         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2112         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2113 }
2114
2115 static void cayman_vce_resume(struct radeon_device *rdev)
2116 {
2117         struct radeon_ring *ring;
2118         int r;
2119
2120         if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2121                 return;
2122
2123         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2124         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2125         if (r) {
2126                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2127                 return;
2128         }
2129         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2130         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2131         if (r) {
2132                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2133                 return;
2134         }
2135         r = vce_v1_0_init(rdev);
2136         if (r) {
2137                 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2138                 return;
2139         }
2140 }
2141
2142 static int cayman_startup(struct radeon_device *rdev)
2143 {
2144         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2145         int r;
2146
2147         /* enable pcie gen2 link */
2148         evergreen_pcie_gen2_enable(rdev);
2149         /* enable aspm */
2150         evergreen_program_aspm(rdev);
2151
2152         /* scratch needs to be initialized before MC */
2153         r = r600_vram_scratch_init(rdev);
2154         if (r)
2155                 return r;
2156
2157         evergreen_mc_program(rdev);
2158
2159         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2160                 r = ni_mc_load_microcode(rdev);
2161                 if (r) {
2162                         DRM_ERROR("Failed to load MC firmware!\n");
2163                         return r;
2164                 }
2165         }
2166
2167         r = cayman_pcie_gart_enable(rdev);
2168         if (r)
2169                 return r;
2170         cayman_gpu_init(rdev);
2171
2172         /* allocate rlc buffers */
2173         if (rdev->flags & RADEON_IS_IGP) {
2174                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2175                 rdev->rlc.reg_list_size =
2176                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2177                 rdev->rlc.cs_data = cayman_cs_data;
2178                 r = sumo_rlc_init(rdev);
2179                 if (r) {
2180                         DRM_ERROR("Failed to init rlc BOs!\n");
2181                         return r;
2182                 }
2183         }
2184
2185         /* allocate wb buffer */
2186         r = radeon_wb_init(rdev);
2187         if (r)
2188                 return r;
2189
2190         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2191         if (r) {
2192                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2193                 return r;
2194         }
2195
2196         cayman_uvd_start(rdev);
2197         cayman_vce_start(rdev);
2198
2199         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2200         if (r) {
2201                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2202                 return r;
2203         }
2204
2205         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2206         if (r) {
2207                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2208                 return r;
2209         }
2210
2211         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2212         if (r) {
2213                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2214                 return r;
2215         }
2216
2217         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2218         if (r) {
2219                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2220                 return r;
2221         }
2222
2223         /* Enable IRQ */
2224         if (!rdev->irq.installed) {
2225                 r = radeon_irq_kms_init(rdev);
2226                 if (r)
2227                         return r;
2228         }
2229
2230         r = r600_irq_init(rdev);
2231         if (r) {
2232                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2233                 radeon_irq_kms_fini(rdev);
2234                 return r;
2235         }
2236         evergreen_irq_set(rdev);
2237
2238         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2239                              RADEON_CP_PACKET2);
2240         if (r)
2241                 return r;
2242
2243         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2244         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2245                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2246         if (r)
2247                 return r;
2248
2249         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2250         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2251                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2252         if (r)
2253                 return r;
2254
2255         r = cayman_cp_load_microcode(rdev);
2256         if (r)
2257                 return r;
2258         r = cayman_cp_resume(rdev);
2259         if (r)
2260                 return r;
2261
2262         r = cayman_dma_resume(rdev);
2263         if (r)
2264                 return r;
2265
2266         cayman_uvd_resume(rdev);
2267         cayman_vce_resume(rdev);
2268
2269         r = radeon_ib_pool_init(rdev);
2270         if (r) {
2271                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2272                 return r;
2273         }
2274
2275         r = radeon_vm_manager_init(rdev);
2276         if (r) {
2277                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2278                 return r;
2279         }
2280
2281         r = radeon_audio_init(rdev);
2282         if (r)
2283                 return r;
2284
2285         return 0;
2286 }
2287
2288 int cayman_resume(struct radeon_device *rdev)
2289 {
2290         int r;
2291
2292         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2293          * posting will perform necessary task to bring back GPU into good
2294          * shape.
2295          */
2296         /* post card */
2297         atom_asic_init(rdev->mode_info.atom_context);
2298
2299         /* init golden registers */
2300         ni_init_golden_registers(rdev);
2301
2302         if (rdev->pm.pm_method == PM_METHOD_DPM)
2303                 radeon_pm_resume(rdev);
2304
2305         rdev->accel_working = true;
2306         r = cayman_startup(rdev);
2307         if (r) {
2308                 DRM_ERROR("cayman startup failed on resume\n");
2309                 rdev->accel_working = false;
2310                 return r;
2311         }
2312         return r;
2313 }
2314
2315 int cayman_suspend(struct radeon_device *rdev)
2316 {
2317         radeon_pm_suspend(rdev);
2318         radeon_audio_fini(rdev);
2319         radeon_vm_manager_fini(rdev);
2320         cayman_cp_enable(rdev, false);
2321         cayman_dma_stop(rdev);
2322         if (rdev->has_uvd) {
2323                 uvd_v1_0_fini(rdev);
2324                 radeon_uvd_suspend(rdev);
2325         }
2326         evergreen_irq_suspend(rdev);
2327         radeon_wb_disable(rdev);
2328         cayman_pcie_gart_disable(rdev);
2329         return 0;
2330 }
2331
2332 /* Plan is to move initialization in that function and use
2333  * helper function so that radeon_device_init pretty much
2334  * do nothing more than calling asic specific function. This
2335  * should also allow to remove a bunch of callback function
2336  * like vram_info.
2337  */
2338 int cayman_init(struct radeon_device *rdev)
2339 {
2340         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2341         int r;
2342
2343         /* Read BIOS */
2344         if (!radeon_get_bios(rdev)) {
2345                 if (ASIC_IS_AVIVO(rdev))
2346                         return -EINVAL;
2347         }
2348         /* Must be an ATOMBIOS */
2349         if (!rdev->is_atom_bios) {
2350                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2351                 return -EINVAL;
2352         }
2353         r = radeon_atombios_init(rdev);
2354         if (r)
2355                 return r;
2356
2357         /* Post card if necessary */
2358         if (!radeon_card_posted(rdev)) {
2359                 if (!rdev->bios) {
2360                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2361                         return -EINVAL;
2362                 }
2363                 DRM_INFO("GPU not posted. posting now...\n");
2364                 atom_asic_init(rdev->mode_info.atom_context);
2365         }
2366         /* init golden registers */
2367         ni_init_golden_registers(rdev);
2368         /* Initialize scratch registers */
2369         r600_scratch_init(rdev);
2370         /* Initialize surface registers */
2371         radeon_surface_init(rdev);
2372         /* Initialize clocks */
2373         radeon_get_clock_info(rdev->ddev);
2374         /* Fence driver */
2375         r = radeon_fence_driver_init(rdev);
2376         if (r)
2377                 return r;
2378         /* initialize memory controller */
2379         r = evergreen_mc_init(rdev);
2380         if (r)
2381                 return r;
2382         /* Memory manager */
2383         r = radeon_bo_init(rdev);
2384         if (r)
2385                 return r;
2386
2387         if (rdev->flags & RADEON_IS_IGP) {
2388                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2389                         r = ni_init_microcode(rdev);
2390                         if (r) {
2391                                 DRM_ERROR("Failed to load firmware!\n");
2392                                 return r;
2393                         }
2394                 }
2395         } else {
2396                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2397                         r = ni_init_microcode(rdev);
2398                         if (r) {
2399                                 DRM_ERROR("Failed to load firmware!\n");
2400                                 return r;
2401                         }
2402                 }
2403         }
2404
2405         /* Initialize power management */
2406         radeon_pm_init(rdev);
2407
2408         ring->ring_obj = NULL;
2409         r600_ring_init(rdev, ring, 1024 * 1024);
2410
2411         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2412         ring->ring_obj = NULL;
2413         r600_ring_init(rdev, ring, 64 * 1024);
2414
2415         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2416         ring->ring_obj = NULL;
2417         r600_ring_init(rdev, ring, 64 * 1024);
2418
2419         cayman_uvd_init(rdev);
2420         cayman_vce_init(rdev);
2421
2422         rdev->ih.ring_obj = NULL;
2423         r600_ih_ring_init(rdev, 64 * 1024);
2424
2425         r = r600_pcie_gart_init(rdev);
2426         if (r)
2427                 return r;
2428
2429         rdev->accel_working = true;
2430         r = cayman_startup(rdev);
2431         if (r) {
2432                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2433                 cayman_cp_fini(rdev);
2434                 cayman_dma_fini(rdev);
2435                 r600_irq_fini(rdev);
2436                 if (rdev->flags & RADEON_IS_IGP)
2437                         sumo_rlc_fini(rdev);
2438                 radeon_wb_fini(rdev);
2439                 radeon_ib_pool_fini(rdev);
2440                 radeon_vm_manager_fini(rdev);
2441                 radeon_irq_kms_fini(rdev);
2442                 cayman_pcie_gart_fini(rdev);
2443                 rdev->accel_working = false;
2444         }
2445
2446         /* Don't start up if the MC ucode is missing.
2447          * The default clocks and voltages before the MC ucode
2448          * is loaded are not suffient for advanced operations.
2449          *
2450          * We can skip this check for TN, because there is no MC
2451          * ucode.
2452          */
2453         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2454                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2455                 return -EINVAL;
2456         }
2457
2458         return 0;
2459 }
2460
2461 void cayman_fini(struct radeon_device *rdev)
2462 {
2463         radeon_pm_fini(rdev);
2464         cayman_cp_fini(rdev);
2465         cayman_dma_fini(rdev);
2466         r600_irq_fini(rdev);
2467         if (rdev->flags & RADEON_IS_IGP)
2468                 sumo_rlc_fini(rdev);
2469         radeon_wb_fini(rdev);
2470         radeon_vm_manager_fini(rdev);
2471         radeon_ib_pool_fini(rdev);
2472         radeon_irq_kms_fini(rdev);
2473         uvd_v1_0_fini(rdev);
2474         radeon_uvd_fini(rdev);
2475         if (rdev->has_vce)
2476                 radeon_vce_fini(rdev);
2477         cayman_pcie_gart_fini(rdev);
2478         r600_vram_scratch_fini(rdev);
2479         radeon_gem_fini(rdev);
2480         radeon_fence_driver_fini(rdev);
2481         radeon_bo_fini(rdev);
2482         radeon_atombios_fini(rdev);
2483         kfree(rdev->bios);
2484         rdev->bios = NULL;
2485 }
2486
2487 /*
2488  * vm
2489  */
2490 int cayman_vm_init(struct radeon_device *rdev)
2491 {
2492         /* number of VMs */
2493         rdev->vm_manager.nvm = 8;
2494         /* base offset of vram pages */
2495         if (rdev->flags & RADEON_IS_IGP) {
2496                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2497                 tmp <<= 22;
2498                 rdev->vm_manager.vram_base_offset = tmp;
2499         } else
2500                 rdev->vm_manager.vram_base_offset = 0;
2501         return 0;
2502 }
2503
2504 void cayman_vm_fini(struct radeon_device *rdev)
2505 {
2506 }
2507
2508 /**
2509  * cayman_vm_decode_fault - print human readable fault info
2510  *
2511  * @rdev: radeon_device pointer
2512  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2513  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2514  *
2515  * Print human readable fault information (cayman/TN).
2516  */
2517 void cayman_vm_decode_fault(struct radeon_device *rdev,
2518                             u32 status, u32 addr)
2519 {
2520         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2521         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2522         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2523         char *block;
2524
2525         switch (mc_id) {
2526         case 32:
2527         case 16:
2528         case 96:
2529         case 80:
2530         case 160:
2531         case 144:
2532         case 224:
2533         case 208:
2534                 block = "CB";
2535                 break;
2536         case 33:
2537         case 17:
2538         case 97:
2539         case 81:
2540         case 161:
2541         case 145:
2542         case 225:
2543         case 209:
2544                 block = "CB_FMASK";
2545                 break;
2546         case 34:
2547         case 18:
2548         case 98:
2549         case 82:
2550         case 162:
2551         case 146:
2552         case 226:
2553         case 210:
2554                 block = "CB_CMASK";
2555                 break;
2556         case 35:
2557         case 19:
2558         case 99:
2559         case 83:
2560         case 163:
2561         case 147:
2562         case 227:
2563         case 211:
2564                 block = "CB_IMMED";
2565                 break;
2566         case 36:
2567         case 20:
2568         case 100:
2569         case 84:
2570         case 164:
2571         case 148:
2572         case 228:
2573         case 212:
2574                 block = "DB";
2575                 break;
2576         case 37:
2577         case 21:
2578         case 101:
2579         case 85:
2580         case 165:
2581         case 149:
2582         case 229:
2583         case 213:
2584                 block = "DB_HTILE";
2585                 break;
2586         case 38:
2587         case 22:
2588         case 102:
2589         case 86:
2590         case 166:
2591         case 150:
2592         case 230:
2593         case 214:
2594                 block = "SX";
2595                 break;
2596         case 39:
2597         case 23:
2598         case 103:
2599         case 87:
2600         case 167:
2601         case 151:
2602         case 231:
2603         case 215:
2604                 block = "DB_STEN";
2605                 break;
2606         case 40:
2607         case 24:
2608         case 104:
2609         case 88:
2610         case 232:
2611         case 216:
2612         case 168:
2613         case 152:
2614                 block = "TC_TFETCH";
2615                 break;
2616         case 41:
2617         case 25:
2618         case 105:
2619         case 89:
2620         case 233:
2621         case 217:
2622         case 169:
2623         case 153:
2624                 block = "TC_VFETCH";
2625                 break;
2626         case 42:
2627         case 26:
2628         case 106:
2629         case 90:
2630         case 234:
2631         case 218:
2632         case 170:
2633         case 154:
2634                 block = "VC";
2635                 break;
2636         case 112:
2637                 block = "CP";
2638                 break;
2639         case 113:
2640         case 114:
2641                 block = "SH";
2642                 break;
2643         case 115:
2644                 block = "VGT";
2645                 break;
2646         case 178:
2647                 block = "IH";
2648                 break;
2649         case 51:
2650                 block = "RLC";
2651                 break;
2652         case 55:
2653                 block = "DMA";
2654                 break;
2655         case 56:
2656                 block = "HDP";
2657                 break;
2658         default:
2659                 block = "unknown";
2660                 break;
2661         }
2662
2663         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2664                protections, vmid, addr,
2665                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2666                block, mc_id);
2667 }
2668
2669 /**
2670  * cayman_vm_flush - vm flush using the CP
2671  *
2672  * @rdev: radeon_device pointer
2673  *
2674  * Update the page table base and flush the VM TLB
2675  * using the CP (cayman-si).
2676  */
2677 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2678                      unsigned vm_id, uint64_t pd_addr)
2679 {
2680         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2681         radeon_ring_write(ring, pd_addr >> 12);
2682
2683         /* flush hdp cache */
2684         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2685         radeon_ring_write(ring, 0x1);
2686
2687         /* bits 0-7 are the VM contexts0-7 */
2688         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2689         radeon_ring_write(ring, 1 << vm_id);
2690
2691         /* wait for the invalidate to complete */
2692         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2693         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2694                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2695         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2696         radeon_ring_write(ring, 0);
2697         radeon_ring_write(ring, 0); /* ref */
2698         radeon_ring_write(ring, 0); /* mask */
2699         radeon_ring_write(ring, 0x20); /* poll interval */
2700
2701         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2702         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2703         radeon_ring_write(ring, 0x0);
2704 }
2705
2706 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2707 {
2708         struct atom_clock_dividers dividers;
2709         int r, i;
2710
2711         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2712                                            ecclk, false, &dividers);
2713         if (r)
2714                 return r;
2715
2716         for (i = 0; i < 100; i++) {
2717                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2718                         break;
2719                 mdelay(10);
2720         }
2721         if (i == 100)
2722                 return -ETIMEDOUT;
2723
2724         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2725
2726         for (i = 0; i < 100; i++) {
2727                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2728                         break;
2729                 mdelay(10);
2730         }
2731         if (i == 100)
2732                 return -ETIMEDOUT;
2733
2734         return 0;
2735 }