drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include <linux/slab.h>
  29
  30 #include <drm/radeon_drm.h>
  31
  32 #include "atom.h"
  33 #include "cayman_blit_shaders.h"
  34 #include "clearstate_cayman.h"
  35 #include "ni_reg.h"
  36 #include "nid.h"
  37 #include "radeon.h"
  38 #include "radeon_asic.h"
  39 #include "radeon_audio.h"
  40 #include "radeon_ucode.h"
  41
  42 /*
  43  * Indirect registers accessor
  44  */
  45 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  46 {
  47         unsigned long flags;
  48         u32 r;
  49
  50         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  51         WREG32(TN_SMC_IND_INDEX_0, (reg));
  52         r = RREG32(TN_SMC_IND_DATA_0);
  53         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  54         return r;
  55 }
  56
  57 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  58 {
  59         unsigned long flags;
  60
  61         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  62         WREG32(TN_SMC_IND_INDEX_0, (reg));
  63         WREG32(TN_SMC_IND_DATA_0, (v));
  64         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  65 }
  66
  67 static const u32 tn_rlc_save_restore_register_list[] =
  68 {
  69         0x98fc,
  70         0x98f0,
  71         0x9834,
  72         0x9838,
  73         0x9870,
  74         0x9874,
  75         0x8a14,
  76         0x8b24,
  77         0x8bcc,
  78         0x8b10,
  79         0x8c30,
  80         0x8d00,
  81         0x8d04,
  82         0x8c00,
  83         0x8c04,
  84         0x8c10,
  85         0x8c14,
  86         0x8d8c,
  87         0x8cf0,
  88         0x8e38,
  89         0x9508,
  90         0x9688,
  91         0x9608,
  92         0x960c,
  93         0x9610,
  94         0x9614,
  95         0x88c4,
  96         0x8978,
  97         0x88d4,
  98         0x900c,
  99         0x9100,
 100         0x913c,
 101         0x90e8,
 102         0x9354,
 103         0xa008,
 104         0x98f8,
 105         0x9148,
 106         0x914c,
 107         0x3f94,
 108         0x98f4,
 109         0x9b7c,
 110         0x3f8c,
 111         0x8950,
 112         0x8954,
 113         0x8a18,
 114         0x8b28,
 115         0x9144,
 116         0x3f90,
 117         0x915c,
 118         0x9160,
 119         0x9178,
 120         0x917c,
 121         0x9180,
 122         0x918c,
 123         0x9190,
 124         0x9194,
 125         0x9198,
 126         0x919c,
 127         0x91a8,
 128         0x91ac,
 129         0x91b0,
 130         0x91b4,
 131         0x91b8,
 132         0x91c4,
 133         0x91c8,
 134         0x91cc,
 135         0x91d0,
 136         0x91d4,
 137         0x91e0,
 138         0x91e4,
 139         0x91ec,
 140         0x91f0,
 141         0x91f4,
 142         0x9200,
 143         0x9204,
 144         0x929c,
 145         0x8030,
 146         0x9150,
 147         0x9a60,
 148         0x920c,
 149         0x9210,
 150         0x9228,
 151         0x922c,
 152         0x9244,
 153         0x9248,
 154         0x91e8,
 155         0x9294,
 156         0x9208,
 157         0x9224,
 158         0x9240,
 159         0x9220,
 160         0x923c,
 161         0x9258,
 162         0x9744,
 163         0xa200,
 164         0xa204,
 165         0xa208,
 166         0xa20c,
 167         0x8d58,
 168         0x9030,
 169         0x9034,
 170         0x9038,
 171         0x903c,
 172         0x9040,
 173         0x9654,
 174         0x897c,
 175         0xa210,
 176         0xa214,
 177         0x9868,
 178         0xa02c,
 179         0x9664,
 180         0x9698,
 181         0x949c,
 182         0x8e10,
 183         0x8e18,
 184         0x8c50,
 185         0x8c58,
 186         0x8c60,
 187         0x8c68,
 188         0x89b4,
 189         0x9830,
 190         0x802c,
 191 };
 192
 193 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 194 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 195 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 196 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 197 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 198 extern void evergreen_mc_program(struct radeon_device *rdev);
 199 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 200 extern int evergreen_mc_init(struct radeon_device *rdev);
 201 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 202 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 203 extern void evergreen_program_aspm(struct radeon_device *rdev);
 204 extern void sumo_rlc_fini(struct radeon_device *rdev);
 205 extern int sumo_rlc_init(struct radeon_device *rdev);
 206 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 207
 208 /* Firmware Names */
 209 /*(DEBLOBBED)*/
 210
 211
 212 static const u32 cayman_golden_registers2[] =
 213 {
 214         0x3e5c, 0xffffffff, 0x00000000,
 215         0x3e48, 0xffffffff, 0x00000000,
 216         0x3e4c, 0xffffffff, 0x00000000,
 217         0x3e64, 0xffffffff, 0x00000000,
 218         0x3e50, 0xffffffff, 0x00000000,
 219         0x3e60, 0xffffffff, 0x00000000
 220 };
 221
 222 static const u32 cayman_golden_registers[] =
 223 {
 224         0x5eb4, 0xffffffff, 0x00000002,
 225         0x5e78, 0x8f311ff1, 0x001000f0,
 226         0x3f90, 0xffff0000, 0xff000000,
 227         0x9148, 0xffff0000, 0xff000000,
 228         0x3f94, 0xffff0000, 0xff000000,
 229         0x914c, 0xffff0000, 0xff000000,
 230         0xc78, 0x00000080, 0x00000080,
 231         0xbd4, 0x70073777, 0x00011003,
 232         0xd02c, 0xbfffff1f, 0x08421000,
 233         0xd0b8, 0x73773777, 0x02011003,
 234         0x5bc0, 0x00200000, 0x50100000,
 235         0x98f8, 0x33773777, 0x02011003,
 236         0x98fc, 0xffffffff, 0x76541032,
 237         0x7030, 0x31000311, 0x00000011,
 238         0x2f48, 0x33773777, 0x42010001,
 239         0x6b28, 0x00000010, 0x00000012,
 240         0x7728, 0x00000010, 0x00000012,
 241         0x10328, 0x00000010, 0x00000012,
 242         0x10f28, 0x00000010, 0x00000012,
 243         0x11b28, 0x00000010, 0x00000012,
 244         0x12728, 0x00000010, 0x00000012,
 245         0x240c, 0x000007ff, 0x00000000,
 246         0x8a14, 0xf000001f, 0x00000007,
 247         0x8b24, 0x3fff3fff, 0x00ff0fff,
 248         0x8b10, 0x0000ff0f, 0x00000000,
 249         0x28a4c, 0x07ffffff, 0x06000000,
 250         0x10c, 0x00000001, 0x00010003,
 251         0xa02c, 0xffffffff, 0x0000009b,
 252         0x913c, 0x0000010f, 0x01000100,
 253         0x8c04, 0xf8ff00ff, 0x40600060,
 254         0x28350, 0x00000f01, 0x00000000,
 255         0x9508, 0x3700001f, 0x00000002,
 256         0x960c, 0xffffffff, 0x54763210,
 257         0x88c4, 0x001f3ae3, 0x00000082,
 258         0x88d0, 0xffffffff, 0x0f40df40,
 259         0x88d4, 0x0000001f, 0x00000010,
 260         0x8974, 0xffffffff, 0x00000000
 261 };
 262
 263 static const u32 dvst_golden_registers2[] =
 264 {
 265         0x8f8, 0xffffffff, 0,
 266         0x8fc, 0x00380000, 0,
 267         0x8f8, 0xffffffff, 1,
 268         0x8fc, 0x0e000000, 0
 269 };
 270
 271 static const u32 dvst_golden_registers[] =
 272 {
 273         0x690, 0x3fff3fff, 0x20c00033,
 274         0x918c, 0x0fff0fff, 0x00010006,
 275         0x91a8, 0x0fff0fff, 0x00010006,
 276         0x9150, 0xffffdfff, 0x6e944040,
 277         0x917c, 0x0fff0fff, 0x00030002,
 278         0x9198, 0x0fff0fff, 0x00030002,
 279         0x915c, 0x0fff0fff, 0x00010000,
 280         0x3f90, 0xffff0001, 0xff000000,
 281         0x9178, 0x0fff0fff, 0x00070000,
 282         0x9194, 0x0fff0fff, 0x00070000,
 283         0x9148, 0xffff0001, 0xff000000,
 284         0x9190, 0x0fff0fff, 0x00090008,
 285         0x91ac, 0x0fff0fff, 0x00090008,
 286         0x3f94, 0xffff0000, 0xff000000,
 287         0x914c, 0xffff0000, 0xff000000,
 288         0x929c, 0x00000fff, 0x00000001,
 289         0x55e4, 0xff607fff, 0xfc000100,
 290         0x8a18, 0xff000fff, 0x00000100,
 291         0x8b28, 0xff000fff, 0x00000100,
 292         0x9144, 0xfffc0fff, 0x00000100,
 293         0x6ed8, 0x00010101, 0x00010000,
 294         0x9830, 0xffffffff, 0x00000000,
 295         0x9834, 0xf00fffff, 0x00000400,
 296         0x9838, 0xfffffffe, 0x00000000,
 297         0xd0c0, 0xff000fff, 0x00000100,
 298         0xd02c, 0xbfffff1f, 0x08421000,
 299         0xd0b8, 0x73773777, 0x12010001,
 300         0x5bb0, 0x000000f0, 0x00000070,
 301         0x98f8, 0x73773777, 0x12010001,
 302         0x98fc, 0xffffffff, 0x00000010,
 303         0x9b7c, 0x00ff0000, 0x00fc0000,
 304         0x8030, 0x00001f0f, 0x0000100a,
 305         0x2f48, 0x73773777, 0x12010001,
 306         0x2408, 0x00030000, 0x000c007f,
 307         0x8a14, 0xf000003f, 0x00000007,
 308         0x8b24, 0x3fff3fff, 0x00ff0fff,
 309         0x8b10, 0x0000ff0f, 0x00000000,
 310         0x28a4c, 0x07ffffff, 0x06000000,
 311         0x4d8, 0x00000fff, 0x00000100,
 312         0xa008, 0xffffffff, 0x00010000,
 313         0x913c, 0xffff03ff, 0x01000100,
 314         0x8c00, 0x000000ff, 0x00000003,
 315         0x8c04, 0xf8ff00ff, 0x40600060,
 316         0x8cf0, 0x1fff1fff, 0x08e00410,
 317         0x28350, 0x00000f01, 0x00000000,
 318         0x9508, 0xf700071f, 0x00000002,
 319         0x960c, 0xffffffff, 0x54763210,
 320         0x20ef8, 0x01ff01ff, 0x00000002,
 321         0x20e98, 0xfffffbff, 0x00200000,
 322         0x2015c, 0xffffffff, 0x00000f40,
 323         0x88c4, 0x001f3ae3, 0x00000082,
 324         0x8978, 0x3fffffff, 0x04050140,
 325         0x88d4, 0x0000001f, 0x00000010,
 326         0x8974, 0xffffffff, 0x00000000
 327 };
 328
 329 static const u32 scrapper_golden_registers[] =
 330 {
 331         0x690, 0x3fff3fff, 0x20c00033,
 332         0x918c, 0x0fff0fff, 0x00010006,
 333         0x918c, 0x0fff0fff, 0x00010006,
 334         0x91a8, 0x0fff0fff, 0x00010006,
 335         0x91a8, 0x0fff0fff, 0x00010006,
 336         0x9150, 0xffffdfff, 0x6e944040,
 337         0x9150, 0xffffdfff, 0x6e944040,
 338         0x917c, 0x0fff0fff, 0x00030002,
 339         0x917c, 0x0fff0fff, 0x00030002,
 340         0x9198, 0x0fff0fff, 0x00030002,
 341         0x9198, 0x0fff0fff, 0x00030002,
 342         0x915c, 0x0fff0fff, 0x00010000,
 343         0x915c, 0x0fff0fff, 0x00010000,
 344         0x3f90, 0xffff0001, 0xff000000,
 345         0x3f90, 0xffff0001, 0xff000000,
 346         0x9178, 0x0fff0fff, 0x00070000,
 347         0x9178, 0x0fff0fff, 0x00070000,
 348         0x9194, 0x0fff0fff, 0x00070000,
 349         0x9194, 0x0fff0fff, 0x00070000,
 350         0x9148, 0xffff0001, 0xff000000,
 351         0x9148, 0xffff0001, 0xff000000,
 352         0x9190, 0x0fff0fff, 0x00090008,
 353         0x9190, 0x0fff0fff, 0x00090008,
 354         0x91ac, 0x0fff0fff, 0x00090008,
 355         0x91ac, 0x0fff0fff, 0x00090008,
 356         0x3f94, 0xffff0000, 0xff000000,
 357         0x3f94, 0xffff0000, 0xff000000,
 358         0x914c, 0xffff0000, 0xff000000,
 359         0x914c, 0xffff0000, 0xff000000,
 360         0x929c, 0x00000fff, 0x00000001,
 361         0x929c, 0x00000fff, 0x00000001,
 362         0x55e4, 0xff607fff, 0xfc000100,
 363         0x8a18, 0xff000fff, 0x00000100,
 364         0x8a18, 0xff000fff, 0x00000100,
 365         0x8b28, 0xff000fff, 0x00000100,
 366         0x8b28, 0xff000fff, 0x00000100,
 367         0x9144, 0xfffc0fff, 0x00000100,
 368         0x9144, 0xfffc0fff, 0x00000100,
 369         0x6ed8, 0x00010101, 0x00010000,
 370         0x9830, 0xffffffff, 0x00000000,
 371         0x9830, 0xffffffff, 0x00000000,
 372         0x9834, 0xf00fffff, 0x00000400,
 373         0x9834, 0xf00fffff, 0x00000400,
 374         0x9838, 0xfffffffe, 0x00000000,
 375         0x9838, 0xfffffffe, 0x00000000,
 376         0xd0c0, 0xff000fff, 0x00000100,
 377         0xd02c, 0xbfffff1f, 0x08421000,
 378         0xd02c, 0xbfffff1f, 0x08421000,
 379         0xd0b8, 0x73773777, 0x12010001,
 380         0xd0b8, 0x73773777, 0x12010001,
 381         0x5bb0, 0x000000f0, 0x00000070,
 382         0x98f8, 0x73773777, 0x12010001,
 383         0x98f8, 0x73773777, 0x12010001,
 384         0x98fc, 0xffffffff, 0x00000010,
 385         0x98fc, 0xffffffff, 0x00000010,
 386         0x9b7c, 0x00ff0000, 0x00fc0000,
 387         0x9b7c, 0x00ff0000, 0x00fc0000,
 388         0x8030, 0x00001f0f, 0x0000100a,
 389         0x8030, 0x00001f0f, 0x0000100a,
 390         0x2f48, 0x73773777, 0x12010001,
 391         0x2f48, 0x73773777, 0x12010001,
 392         0x2408, 0x00030000, 0x000c007f,
 393         0x8a14, 0xf000003f, 0x00000007,
 394         0x8a14, 0xf000003f, 0x00000007,
 395         0x8b24, 0x3fff3fff, 0x00ff0fff,
 396         0x8b24, 0x3fff3fff, 0x00ff0fff,
 397         0x8b10, 0x0000ff0f, 0x00000000,
 398         0x8b10, 0x0000ff0f, 0x00000000,
 399         0x28a4c, 0x07ffffff, 0x06000000,
 400         0x28a4c, 0x07ffffff, 0x06000000,
 401         0x4d8, 0x00000fff, 0x00000100,
 402         0x4d8, 0x00000fff, 0x00000100,
 403         0xa008, 0xffffffff, 0x00010000,
 404         0xa008, 0xffffffff, 0x00010000,
 405         0x913c, 0xffff03ff, 0x01000100,
 406         0x913c, 0xffff03ff, 0x01000100,
 407         0x90e8, 0x001fffff, 0x010400c0,
 408         0x8c00, 0x000000ff, 0x00000003,
 409         0x8c00, 0x000000ff, 0x00000003,
 410         0x8c04, 0xf8ff00ff, 0x40600060,
 411         0x8c04, 0xf8ff00ff, 0x40600060,
 412         0x8c30, 0x0000000f, 0x00040005,
 413         0x8cf0, 0x1fff1fff, 0x08e00410,
 414         0x8cf0, 0x1fff1fff, 0x08e00410,
 415         0x900c, 0x00ffffff, 0x0017071f,
 416         0x28350, 0x00000f01, 0x00000000,
 417         0x28350, 0x00000f01, 0x00000000,
 418         0x9508, 0xf700071f, 0x00000002,
 419         0x9508, 0xf700071f, 0x00000002,
 420         0x9688, 0x00300000, 0x0017000f,
 421         0x960c, 0xffffffff, 0x54763210,
 422         0x960c, 0xffffffff, 0x54763210,
 423         0x20ef8, 0x01ff01ff, 0x00000002,
 424         0x20e98, 0xfffffbff, 0x00200000,
 425         0x2015c, 0xffffffff, 0x00000f40,
 426         0x88c4, 0x001f3ae3, 0x00000082,
 427         0x88c4, 0x001f3ae3, 0x00000082,
 428         0x8978, 0x3fffffff, 0x04050140,
 429         0x8978, 0x3fffffff, 0x04050140,
 430         0x88d4, 0x0000001f, 0x00000010,
 431         0x88d4, 0x0000001f, 0x00000010,
 432         0x8974, 0xffffffff, 0x00000000,
 433         0x8974, 0xffffffff, 0x00000000
 434 };
 435
 436 static void ni_init_golden_registers(struct radeon_device *rdev)
 437 {
 438         switch (rdev->family) {
 439         case CHIP_CAYMAN:
 440                 radeon_program_register_sequence(rdev,
 441                                                  cayman_golden_registers,
 442                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 443                 radeon_program_register_sequence(rdev,
 444                                                  cayman_golden_registers2,
 445                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 446                 break;
 447         case CHIP_ARUBA:
 448                 if ((rdev->pdev->device == 0x9900) ||
 449                     (rdev->pdev->device == 0x9901) ||
 450                     (rdev->pdev->device == 0x9903) ||
 451                     (rdev->pdev->device == 0x9904) ||
 452                     (rdev->pdev->device == 0x9905) ||
 453                     (rdev->pdev->device == 0x9906) ||
 454                     (rdev->pdev->device == 0x9907) ||
 455                     (rdev->pdev->device == 0x9908) ||
 456                     (rdev->pdev->device == 0x9909) ||
 457                     (rdev->pdev->device == 0x990A) ||
 458                     (rdev->pdev->device == 0x990B) ||
 459                     (rdev->pdev->device == 0x990C) ||
 460                     (rdev->pdev->device == 0x990D) ||
 461                     (rdev->pdev->device == 0x990E) ||
 462                     (rdev->pdev->device == 0x990F) ||
 463                     (rdev->pdev->device == 0x9910) ||
 464                     (rdev->pdev->device == 0x9913) ||
 465                     (rdev->pdev->device == 0x9917) ||
 466                     (rdev->pdev->device == 0x9918)) {
 467                         radeon_program_register_sequence(rdev,
 468                                                          dvst_golden_registers,
 469                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 470                         radeon_program_register_sequence(rdev,
 471                                                          dvst_golden_registers2,
 472                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 473                 } else {
 474                         radeon_program_register_sequence(rdev,
 475                                                          scrapper_golden_registers,
 476                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 477                         radeon_program_register_sequence(rdev,
 478                                                          dvst_golden_registers2,
 479                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 480                 }
 481                 break;
 482         default:
 483                 break;
 484         }
 485 }
 486
 487 #define BTC_IO_MC_REGS_SIZE 29
 488
 489 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 490         {0x00000077, 0xff010100},
 491         {0x00000078, 0x00000000},
 492         {0x00000079, 0x00001434},
 493         {0x0000007a, 0xcc08ec08},
 494         {0x0000007b, 0x00040000},
 495         {0x0000007c, 0x000080c0},
 496         {0x0000007d, 0x09000000},
 497         {0x0000007e, 0x00210404},
 498         {0x00000081, 0x08a8e800},
 499         {0x00000082, 0x00030444},
 500         {0x00000083, 0x00000000},
 501         {0x00000085, 0x00000001},
 502         {0x00000086, 0x00000002},
 503         {0x00000087, 0x48490000},
 504         {0x00000088, 0x20244647},
 505         {0x00000089, 0x00000005},
 506         {0x0000008b, 0x66030000},
 507         {0x0000008c, 0x00006603},
 508         {0x0000008d, 0x00000100},
 509         {0x0000008f, 0x00001c0a},
 510         {0x00000090, 0xff000001},
 511         {0x00000094, 0x00101101},
 512         {0x00000095, 0x00000fff},
 513         {0x00000096, 0x00116fff},
 514         {0x00000097, 0x60010000},
 515         {0x00000098, 0x10010000},
 516         {0x00000099, 0x00006000},
 517         {0x0000009a, 0x00001000},
 518         {0x0000009f, 0x00946a00}
 519 };
 520
 521 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 522         {0x00000077, 0xff010100},
 523         {0x00000078, 0x00000000},
 524         {0x00000079, 0x00001434},
 525         {0x0000007a, 0xcc08ec08},
 526         {0x0000007b, 0x00040000},
 527         {0x0000007c, 0x000080c0},
 528         {0x0000007d, 0x09000000},
 529         {0x0000007e, 0x00210404},
 530         {0x00000081, 0x08a8e800},
 531         {0x00000082, 0x00030444},
 532         {0x00000083, 0x00000000},
 533         {0x00000085, 0x00000001},
 534         {0x00000086, 0x00000002},
 535         {0x00000087, 0x48490000},
 536         {0x00000088, 0x20244647},
 537         {0x00000089, 0x00000005},
 538         {0x0000008b, 0x66030000},
 539         {0x0000008c, 0x00006603},
 540         {0x0000008d, 0x00000100},
 541         {0x0000008f, 0x00001c0a},
 542         {0x00000090, 0xff000001},
 543         {0x00000094, 0x00101101},
 544         {0x00000095, 0x00000fff},
 545         {0x00000096, 0x00116fff},
 546         {0x00000097, 0x60010000},
 547         {0x00000098, 0x10010000},
 548         {0x00000099, 0x00006000},
 549         {0x0000009a, 0x00001000},
 550         {0x0000009f, 0x00936a00}
 551 };
 552
 553 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 554         {0x00000077, 0xff010100},
 555         {0x00000078, 0x00000000},
 556         {0x00000079, 0x00001434},
 557         {0x0000007a, 0xcc08ec08},
 558         {0x0000007b, 0x00040000},
 559         {0x0000007c, 0x000080c0},
 560         {0x0000007d, 0x09000000},
 561         {0x0000007e, 0x00210404},
 562         {0x00000081, 0x08a8e800},
 563         {0x00000082, 0x00030444},
 564         {0x00000083, 0x00000000},
 565         {0x00000085, 0x00000001},
 566         {0x00000086, 0x00000002},
 567         {0x00000087, 0x48490000},
 568         {0x00000088, 0x20244647},
 569         {0x00000089, 0x00000005},
 570         {0x0000008b, 0x66030000},
 571         {0x0000008c, 0x00006603},
 572         {0x0000008d, 0x00000100},
 573         {0x0000008f, 0x00001c0a},
 574         {0x00000090, 0xff000001},
 575         {0x00000094, 0x00101101},
 576         {0x00000095, 0x00000fff},
 577         {0x00000096, 0x00116fff},
 578         {0x00000097, 0x60010000},
 579         {0x00000098, 0x10010000},
 580         {0x00000099, 0x00006000},
 581         {0x0000009a, 0x00001000},
 582         {0x0000009f, 0x00916a00}
 583 };
 584
 585 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 586         {0x00000077, 0xff010100},
 587         {0x00000078, 0x00000000},
 588         {0x00000079, 0x00001434},
 589         {0x0000007a, 0xcc08ec08},
 590         {0x0000007b, 0x00040000},
 591         {0x0000007c, 0x000080c0},
 592         {0x0000007d, 0x09000000},
 593         {0x0000007e, 0x00210404},
 594         {0x00000081, 0x08a8e800},
 595         {0x00000082, 0x00030444},
 596         {0x00000083, 0x00000000},
 597         {0x00000085, 0x00000001},
 598         {0x00000086, 0x00000002},
 599         {0x00000087, 0x48490000},
 600         {0x00000088, 0x20244647},
 601         {0x00000089, 0x00000005},
 602         {0x0000008b, 0x66030000},
 603         {0x0000008c, 0x00006603},
 604         {0x0000008d, 0x00000100},
 605         {0x0000008f, 0x00001c0a},
 606         {0x00000090, 0xff000001},
 607         {0x00000094, 0x00101101},
 608         {0x00000095, 0x00000fff},
 609         {0x00000096, 0x00116fff},
 610         {0x00000097, 0x60010000},
 611         {0x00000098, 0x10010000},
 612         {0x00000099, 0x00006000},
 613         {0x0000009a, 0x00001000},
 614         {0x0000009f, 0x00976b00}
 615 };
 616
 617 int ni_mc_load_microcode(struct radeon_device *rdev)
 618 {
 619         const __be32 *fw_data;
 620         u32 mem_type, running, blackout = 0;
 621         u32 *io_mc_regs;
 622         int i, ucode_size, regs_size;
 623
 624         if (!rdev->mc_fw)
 625                 return -EINVAL;
 626
 627         switch (rdev->family) {
 628         case CHIP_BARTS:
 629                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 630                 ucode_size = BTC_MC_UCODE_SIZE;
 631                 regs_size = BTC_IO_MC_REGS_SIZE;
 632                 break;
 633         case CHIP_TURKS:
 634                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 635                 ucode_size = BTC_MC_UCODE_SIZE;
 636                 regs_size = BTC_IO_MC_REGS_SIZE;
 637                 break;
 638         case CHIP_CAICOS:
 639         default:
 640                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 641                 ucode_size = BTC_MC_UCODE_SIZE;
 642                 regs_size = BTC_IO_MC_REGS_SIZE;
 643                 break;
 644         case CHIP_CAYMAN:
 645                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 646                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 647                 regs_size = BTC_IO_MC_REGS_SIZE;
 648                 break;
 649         }
 650
 651         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 652         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 653
 654         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 655                 if (running) {
 656                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 657                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 658                 }
 659
 660                 /* reset the engine and set to writable */
 661                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 662                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 663
 664                 /* load mc io regs */
 665                 for (i = 0; i < regs_size; i++) {
 666                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 667                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 668                 }
 669                 /* load the MC ucode */
 670                 fw_data = (const __be32 *)rdev->mc_fw->data;
 671                 for (i = 0; i < ucode_size; i++)
 672                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 673
 674                 /* put the engine back into the active state */
 675                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 676                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 677                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 678
 679                 /* wait for training to complete */
 680                 for (i = 0; i < rdev->usec_timeout; i++) {
 681                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 682                                 break;
 683                         udelay(1);
 684                 }
 685
 686                 if (running)
 687                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 688         }
 689
 690         return 0;
 691 }
 692
 693 int ni_init_microcode(struct radeon_device *rdev)
 694 {
 695         const char *chip_name;
 696         const char *rlc_chip_name;
 697         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 698         size_t smc_req_size = 0;
 699         char fw_name[30];
 700         int err;
 701
 702         DRM_DEBUG("\n");
 703
 704         switch (rdev->family) {
 705         case CHIP_BARTS:
 706                 chip_name = "BARTS";
 707                 rlc_chip_name = "BTC";
 708                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 709                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 710                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 711                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 712                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 713                 break;
 714         case CHIP_TURKS:
 715                 chip_name = "TURKS";
 716                 rlc_chip_name = "BTC";
 717                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 718                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 719                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 720                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 721                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 722                 break;
 723         case CHIP_CAICOS:
 724                 chip_name = "CAICOS";
 725                 rlc_chip_name = "BTC";
 726                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 727                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 728                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 729                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 730                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 731                 break;
 732         case CHIP_CAYMAN:
 733                 chip_name = "CAYMAN";
 734                 rlc_chip_name = "CAYMAN";
 735                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 736                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 737                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 738                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 739                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 740                 break;
 741         case CHIP_ARUBA:
 742                 chip_name = "ARUBA";
 743                 rlc_chip_name = "ARUBA";
 744                 /* pfp/me same size as CAYMAN */
 745                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 746                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 747                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 748                 mc_req_size = 0;
 749                 break;
 750         default: BUG();
 751         }
 752
 753         DRM_INFO("Loading %s Microcode\n", chip_name);
 754
 755         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 756         err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 757         if (err)
 758                 goto out;
 759         if (rdev->pfp_fw->size != pfp_req_size) {
 760                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 761                        rdev->pfp_fw->size, fw_name);
 762                 err = -EINVAL;
 763                 goto out;
 764         }
 765
 766         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 767         err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
 768         if (err)
 769                 goto out;
 770         if (rdev->me_fw->size != me_req_size) {
 771                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 772                        rdev->me_fw->size, fw_name);
 773                 err = -EINVAL;
 774         }
 775
 776         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", rlc_chip_name);
 777         err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 778         if (err)
 779                 goto out;
 780         if (rdev->rlc_fw->size != rlc_req_size) {
 781                 pr_err("ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 782                        rdev->rlc_fw->size, fw_name);
 783                 err = -EINVAL;
 784         }
 785
 786         /* no MC ucode on TN */
 787         if (!(rdev->flags & RADEON_IS_IGP)) {
 788                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 789                 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 790                 if (err)
 791                         goto out;
 792                 if (rdev->mc_fw->size != mc_req_size) {
 793                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 794                                rdev->mc_fw->size, fw_name);
 795                         err = -EINVAL;
 796                 }
 797         }
 798
 799         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 800                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 801                 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 802                 if (err) {
 803                         pr_err("smc: error loading firmware \"%s\"\n", fw_name);
 804                         release_firmware(rdev->smc_fw);
 805                         rdev->smc_fw = NULL;
 806                         err = 0;
 807                 } else if (rdev->smc_fw->size != smc_req_size) {
 808                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 809                                rdev->mc_fw->size, fw_name);
 810                         err = -EINVAL;
 811                 }
 812         }
 813
 814 out:
 815         if (err) {
 816                 if (err != -EINVAL)
 817                         pr_err("ni_cp: Failed to load firmware \"%s\"\n",
 818                                fw_name);
 819                 release_firmware(rdev->pfp_fw);
 820                 rdev->pfp_fw = NULL;
 821                 release_firmware(rdev->me_fw);
 822                 rdev->me_fw = NULL;
 823                 release_firmware(rdev->rlc_fw);
 824                 rdev->rlc_fw = NULL;
 825                 release_firmware(rdev->mc_fw);
 826                 rdev->mc_fw = NULL;
 827         }
 828         return err;
 829 }
 830
 831 /**
 832  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 833  *
 834  * @rdev: radeon_device pointer
 835  * @reg: register offset in bytes
 836  * @val: register value
 837  *
 838  * Returns 0 for success or -EINVAL for an invalid register
 839  *
 840  */
 841 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 842                                      u32 reg, u32 *val)
 843 {
 844         switch (reg) {
 845         case GRBM_STATUS:
 846         case GRBM_STATUS_SE0:
 847         case GRBM_STATUS_SE1:
 848         case SRBM_STATUS:
 849         case SRBM_STATUS2:
 850         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 851         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 852         case UVD_STATUS:
 853                 *val = RREG32(reg);
 854                 return 0;
 855         default:
 856                 return -EINVAL;
 857         }
 858 }
 859
 860 int tn_get_temp(struct radeon_device *rdev)
 861 {
 862         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 863         int actual_temp = (temp / 8) - 49;
 864
 865         return actual_temp * 1000;
 866 }
 867
 868 /*
 869  * Core functions
 870  */
 871 static void cayman_gpu_init(struct radeon_device *rdev)
 872 {
 873         u32 gb_addr_config = 0;
 874         u32 mc_shared_chmap, mc_arb_ramcfg;
 875         u32 cgts_tcc_disable;
 876         u32 sx_debug_1;
 877         u32 smx_dc_ctl0;
 878         u32 cgts_sm_ctrl_reg;
 879         u32 hdp_host_path_cntl;
 880         u32 tmp;
 881         u32 disabled_rb_mask;
 882         int i, j;
 883
 884         switch (rdev->family) {
 885         case CHIP_CAYMAN:
 886                 rdev->config.cayman.max_shader_engines = 2;
 887                 rdev->config.cayman.max_pipes_per_simd = 4;
 888                 rdev->config.cayman.max_tile_pipes = 8;
 889                 rdev->config.cayman.max_simds_per_se = 12;
 890                 rdev->config.cayman.max_backends_per_se = 4;
 891                 rdev->config.cayman.max_texture_channel_caches = 8;
 892                 rdev->config.cayman.max_gprs = 256;
 893                 rdev->config.cayman.max_threads = 256;
 894                 rdev->config.cayman.max_gs_threads = 32;
 895                 rdev->config.cayman.max_stack_entries = 512;
 896                 rdev->config.cayman.sx_num_of_sets = 8;
 897                 rdev->config.cayman.sx_max_export_size = 256;
 898                 rdev->config.cayman.sx_max_export_pos_size = 64;
 899                 rdev->config.cayman.sx_max_export_smx_size = 192;
 900                 rdev->config.cayman.max_hw_contexts = 8;
 901                 rdev->config.cayman.sq_num_cf_insts = 2;
 902
 903                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 904                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 905                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 906                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 907                 break;
 908         case CHIP_ARUBA:
 909         default:
 910                 rdev->config.cayman.max_shader_engines = 1;
 911                 rdev->config.cayman.max_pipes_per_simd = 4;
 912                 rdev->config.cayman.max_tile_pipes = 2;
 913                 if ((rdev->pdev->device == 0x9900) ||
 914                     (rdev->pdev->device == 0x9901) ||
 915                     (rdev->pdev->device == 0x9905) ||
 916                     (rdev->pdev->device == 0x9906) ||
 917                     (rdev->pdev->device == 0x9907) ||
 918                     (rdev->pdev->device == 0x9908) ||
 919                     (rdev->pdev->device == 0x9909) ||
 920                     (rdev->pdev->device == 0x990B) ||
 921                     (rdev->pdev->device == 0x990C) ||
 922                     (rdev->pdev->device == 0x990F) ||
 923                     (rdev->pdev->device == 0x9910) ||
 924                     (rdev->pdev->device == 0x9917) ||
 925                     (rdev->pdev->device == 0x9999) ||
 926                     (rdev->pdev->device == 0x999C)) {
 927                         rdev->config.cayman.max_simds_per_se = 6;
 928                         rdev->config.cayman.max_backends_per_se = 2;
 929                         rdev->config.cayman.max_hw_contexts = 8;
 930                         rdev->config.cayman.sx_max_export_size = 256;
 931                         rdev->config.cayman.sx_max_export_pos_size = 64;
 932                         rdev->config.cayman.sx_max_export_smx_size = 192;
 933                 } else if ((rdev->pdev->device == 0x9903) ||
 934                            (rdev->pdev->device == 0x9904) ||
 935                            (rdev->pdev->device == 0x990A) ||
 936                            (rdev->pdev->device == 0x990D) ||
 937                            (rdev->pdev->device == 0x990E) ||
 938                            (rdev->pdev->device == 0x9913) ||
 939                            (rdev->pdev->device == 0x9918) ||
 940                            (rdev->pdev->device == 0x999D)) {
 941                         rdev->config.cayman.max_simds_per_se = 4;
 942                         rdev->config.cayman.max_backends_per_se = 2;
 943                         rdev->config.cayman.max_hw_contexts = 8;
 944                         rdev->config.cayman.sx_max_export_size = 256;
 945                         rdev->config.cayman.sx_max_export_pos_size = 64;
 946                         rdev->config.cayman.sx_max_export_smx_size = 192;
 947                 } else if ((rdev->pdev->device == 0x9919) ||
 948                            (rdev->pdev->device == 0x9990) ||
 949                            (rdev->pdev->device == 0x9991) ||
 950                            (rdev->pdev->device == 0x9994) ||
 951                            (rdev->pdev->device == 0x9995) ||
 952                            (rdev->pdev->device == 0x9996) ||
 953                            (rdev->pdev->device == 0x999A) ||
 954                            (rdev->pdev->device == 0x99A0)) {
 955                         rdev->config.cayman.max_simds_per_se = 3;
 956                         rdev->config.cayman.max_backends_per_se = 1;
 957                         rdev->config.cayman.max_hw_contexts = 4;
 958                         rdev->config.cayman.sx_max_export_size = 128;
 959                         rdev->config.cayman.sx_max_export_pos_size = 32;
 960                         rdev->config.cayman.sx_max_export_smx_size = 96;
 961                 } else {
 962                         rdev->config.cayman.max_simds_per_se = 2;
 963                         rdev->config.cayman.max_backends_per_se = 1;
 964                         rdev->config.cayman.max_hw_contexts = 4;
 965                         rdev->config.cayman.sx_max_export_size = 128;
 966                         rdev->config.cayman.sx_max_export_pos_size = 32;
 967                         rdev->config.cayman.sx_max_export_smx_size = 96;
 968                 }
 969                 rdev->config.cayman.max_texture_channel_caches = 2;
 970                 rdev->config.cayman.max_gprs = 256;
 971                 rdev->config.cayman.max_threads = 256;
 972                 rdev->config.cayman.max_gs_threads = 32;
 973                 rdev->config.cayman.max_stack_entries = 512;
 974                 rdev->config.cayman.sx_num_of_sets = 8;
 975                 rdev->config.cayman.sq_num_cf_insts = 2;
 976
 977                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 978                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 979                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 980                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 981                 break;
 982         }
 983
 984         /* Initialize HDP */
 985         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 986                 WREG32((0x2c14 + j), 0x00000000);
 987                 WREG32((0x2c18 + j), 0x00000000);
 988                 WREG32((0x2c1c + j), 0x00000000);
 989                 WREG32((0x2c20 + j), 0x00000000);
 990                 WREG32((0x2c24 + j), 0x00000000);
 991         }
 992
 993         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 994         WREG32(SRBM_INT_CNTL, 0x1);
 995         WREG32(SRBM_INT_ACK, 0x1);
 996
 997         evergreen_fix_pci_max_read_req_size(rdev);
 998
 999         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1000         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1001
1002         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1003         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1004         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1005                 rdev->config.cayman.mem_row_size_in_kb = 4;
1006         /* XXX use MC settings? */
1007         rdev->config.cayman.shader_engine_tile_size = 32;
1008         rdev->config.cayman.num_gpus = 1;
1009         rdev->config.cayman.multi_gpu_tile_size = 64;
1010
1011         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1012         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1013         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1014         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1015         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1016         rdev->config.cayman.num_shader_engines = tmp + 1;
1017         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1018         rdev->config.cayman.num_gpus = tmp + 1;
1019         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1020         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1021         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1022         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1023
1024
1025         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1026          * not have bank info, so create a custom tiling dword.
1027          * bits 3:0   num_pipes
1028          * bits 7:4   num_banks
1029          * bits 11:8  group_size
1030          * bits 15:12 row_size
1031          */
1032         rdev->config.cayman.tile_config = 0;
1033         switch (rdev->config.cayman.num_tile_pipes) {
1034         case 1:
1035         default:
1036                 rdev->config.cayman.tile_config |= (0 << 0);
1037                 break;
1038         case 2:
1039                 rdev->config.cayman.tile_config |= (1 << 0);
1040                 break;
1041         case 4:
1042                 rdev->config.cayman.tile_config |= (2 << 0);
1043                 break;
1044         case 8:
1045                 rdev->config.cayman.tile_config |= (3 << 0);
1046                 break;
1047         }
1048
1049         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1050         if (rdev->flags & RADEON_IS_IGP)
1051                 rdev->config.cayman.tile_config |= 1 << 4;
1052         else {
1053                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1054                 case 0: /* four banks */
1055                         rdev->config.cayman.tile_config |= 0 << 4;
1056                         break;
1057                 case 1: /* eight banks */
1058                         rdev->config.cayman.tile_config |= 1 << 4;
1059                         break;
1060                 case 2: /* sixteen banks */
1061                 default:
1062                         rdev->config.cayman.tile_config |= 2 << 4;
1063                         break;
1064                 }
1065         }
1066         rdev->config.cayman.tile_config |=
1067                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1068         rdev->config.cayman.tile_config |=
1069                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1070
1071         tmp = 0;
1072         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1073                 u32 rb_disable_bitmap;
1074
1075                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1076                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1077                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1078                 tmp <<= 4;
1079                 tmp |= rb_disable_bitmap;
1080         }
1081         /* enabled rb are just the one not disabled :) */
1082         disabled_rb_mask = tmp;
1083         tmp = 0;
1084         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1085                 tmp |= (1 << i);
1086         /* if all the backends are disabled, fix it up here */
1087         if ((disabled_rb_mask & tmp) == tmp) {
1088                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1089                         disabled_rb_mask &= ~(1 << i);
1090         }
1091
1092         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1093                 u32 simd_disable_bitmap;
1094
1095                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1096                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1097                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1098                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1099                 tmp <<= 16;
1100                 tmp |= simd_disable_bitmap;
1101         }
1102         rdev->config.cayman.active_simds = hweight32(~tmp);
1103
1104         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1105         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1106
1107         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1108         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1109         if (ASIC_IS_DCE6(rdev))
1110                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1111         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1112         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1113         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1114         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1115         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1116         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1117
1118         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1119             (rdev->flags & RADEON_IS_IGP)) {
1120                 if ((disabled_rb_mask & 3) == 2) {
1121                         /* RB1 disabled, RB0 enabled */
1122                         tmp = 0x00000000;
1123                 } else {
1124                         /* RB0 disabled, RB1 enabled */
1125                         tmp = 0x11111111;
1126                 }
1127         } else {
1128                 tmp = gb_addr_config & NUM_PIPES_MASK;
1129                 tmp = r6xx_remap_render_backend(rdev, tmp,
1130                                                 rdev->config.cayman.max_backends_per_se *
1131                                                 rdev->config.cayman.max_shader_engines,
1132                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1133         }
1134         rdev->config.cayman.backend_map = tmp;
1135         WREG32(GB_BACKEND_MAP, tmp);
1136
1137         cgts_tcc_disable = 0xffff0000;
1138         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1139                 cgts_tcc_disable &= ~(1 << (16 + i));
1140         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1141         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1142         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1143         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1144
1145         /* reprogram the shader complex */
1146         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1147         for (i = 0; i < 16; i++)
1148                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1149         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1150
1151         /* set HW defaults for 3D engine */
1152         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1153
1154         sx_debug_1 = RREG32(SX_DEBUG_1);
1155         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1156         WREG32(SX_DEBUG_1, sx_debug_1);
1157
1158         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1159         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1160         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1161         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1162
1163         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1164
1165         /* need to be explicitly zero-ed */
1166         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1167         WREG32(SQ_LSTMP_RING_BASE, 0);
1168         WREG32(SQ_HSTMP_RING_BASE, 0);
1169         WREG32(SQ_ESTMP_RING_BASE, 0);
1170         WREG32(SQ_GSTMP_RING_BASE, 0);
1171         WREG32(SQ_VSTMP_RING_BASE, 0);
1172         WREG32(SQ_PSTMP_RING_BASE, 0);
1173
1174         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1175
1176         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1177                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1178                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1179
1180         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1181                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1182                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1183
1184
1185         WREG32(VGT_NUM_INSTANCES, 1);
1186
1187         WREG32(CP_PERFMON_CNTL, 0);
1188
1189         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1190                                   FETCH_FIFO_HIWATER(0x4) |
1191                                   DONE_FIFO_HIWATER(0xe0) |
1192                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1193
1194         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1195         WREG32(SQ_CONFIG, (VC_ENABLE |
1196                            EXPORT_SRC_C |
1197                            GFX_PRIO(0) |
1198                            CS1_PRIO(0) |
1199                            CS2_PRIO(1)));
1200         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1201
1202         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1203                                           FORCE_EOV_MAX_REZ_CNT(255)));
1204
1205         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1206                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1207
1208         WREG32(VGT_GS_VERTEX_REUSE, 16);
1209         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1210
1211         WREG32(CB_PERF_CTR0_SEL_0, 0);
1212         WREG32(CB_PERF_CTR0_SEL_1, 0);
1213         WREG32(CB_PERF_CTR1_SEL_0, 0);
1214         WREG32(CB_PERF_CTR1_SEL_1, 0);
1215         WREG32(CB_PERF_CTR2_SEL_0, 0);
1216         WREG32(CB_PERF_CTR2_SEL_1, 0);
1217         WREG32(CB_PERF_CTR3_SEL_0, 0);
1218         WREG32(CB_PERF_CTR3_SEL_1, 0);
1219
1220         tmp = RREG32(HDP_MISC_CNTL);
1221         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1222         WREG32(HDP_MISC_CNTL, tmp);
1223
1224         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1225         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1226
1227         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1228
1229         udelay(50);
1230
1231         /* set clockgating golden values on TN */
1232         if (rdev->family == CHIP_ARUBA) {
1233                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1234                 tmp &= ~0x00380000;
1235                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1236                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1237                 tmp &= ~0x0e000000;
1238                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1239         }
1240 }
1241
1242 /*
1243  * GART
1244  */
1245 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1246 {
1247         /* flush hdp cache */
1248         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1249
1250         /* bits 0-7 are the VM contexts0-7 */
1251         WREG32(VM_INVALIDATE_REQUEST, 1);
1252 }
1253
1254 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1255 {
1256         int i, r;
1257
1258         if (rdev->gart.robj == NULL) {
1259                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1260                 return -EINVAL;
1261         }
1262         r = radeon_gart_table_vram_pin(rdev);
1263         if (r)
1264                 return r;
1265         /* Setup TLB control */
1266         WREG32(MC_VM_MX_L1_TLB_CNTL,
1267                (0xA << 7) |
1268                ENABLE_L1_TLB |
1269                ENABLE_L1_FRAGMENT_PROCESSING |
1270                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1271                ENABLE_ADVANCED_DRIVER_MODEL |
1272                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1273         /* Setup L2 cache */
1274         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1275                ENABLE_L2_FRAGMENT_PROCESSING |
1276                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1277                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1278                EFFECTIVE_L2_QUEUE_SIZE(7) |
1279                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1280         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1281         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1282                BANK_SELECT(6) |
1283                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1284         /* setup context0 */
1285         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1286         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1287         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1288         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1289                         (u32)(rdev->dummy_page.addr >> 12));
1290         WREG32(VM_CONTEXT0_CNTL2, 0);
1291         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1292                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1293
1294         WREG32(0x15D4, 0);
1295         WREG32(0x15D8, 0);
1296         WREG32(0x15DC, 0);
1297
1298         /* empty context1-7 */
1299         /* Assign the pt base to something valid for now; the pts used for
1300          * the VMs are determined by the application and setup and assigned
1301          * on the fly in the vm part of radeon_gart.c
1302          */
1303         for (i = 1; i < 8; i++) {
1304                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1305                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1306                         rdev->vm_manager.max_pfn - 1);
1307                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1308                        rdev->vm_manager.saved_table_addr[i]);
1309         }
1310
1311         /* enable context1-7 */
1312         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1313                (u32)(rdev->dummy_page.addr >> 12));
1314         WREG32(VM_CONTEXT1_CNTL2, 4);
1315         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1316                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1317                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1318                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1319                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1320                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1321                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1322                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1323                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1324                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1325                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1326                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1327                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1328                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1329
1330         cayman_pcie_gart_tlb_flush(rdev);
1331         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1332                  (unsigned)(rdev->mc.gtt_size >> 20),
1333                  (unsigned long long)rdev->gart.table_addr);
1334         rdev->gart.ready = true;
1335         return 0;
1336 }
1337
1338 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1339 {
1340         unsigned i;
1341
1342         for (i = 1; i < 8; ++i) {
1343                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1344                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1345         }
1346
1347         /* Disable all tables */
1348         WREG32(VM_CONTEXT0_CNTL, 0);
1349         WREG32(VM_CONTEXT1_CNTL, 0);
1350         /* Setup TLB control */
1351         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1352                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1353                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1354         /* Setup L2 cache */
1355         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1356                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1357                EFFECTIVE_L2_QUEUE_SIZE(7) |
1358                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1359         WREG32(VM_L2_CNTL2, 0);
1360         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1361                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1362         radeon_gart_table_vram_unpin(rdev);
1363 }
1364
1365 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1366 {
1367         cayman_pcie_gart_disable(rdev);
1368         radeon_gart_table_vram_free(rdev);
1369         radeon_gart_fini(rdev);
1370 }
1371
1372 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1373                               int ring, u32 cp_int_cntl)
1374 {
1375         WREG32(SRBM_GFX_CNTL, RINGID(ring));
1376         WREG32(CP_INT_CNTL, cp_int_cntl);
1377 }
1378
1379 /*
1380  * CP.
1381  */
1382 void cayman_fence_ring_emit(struct radeon_device *rdev,
1383                             struct radeon_fence *fence)
1384 {
1385         struct radeon_ring *ring = &rdev->ring[fence->ring];
1386         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1387         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1388                 PACKET3_SH_ACTION_ENA;
1389
1390         /* flush read cache over gart for this vmid */
1391         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1392         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1393         radeon_ring_write(ring, 0xFFFFFFFF);
1394         radeon_ring_write(ring, 0);
1395         radeon_ring_write(ring, 10); /* poll interval */
1396         /* EVENT_WRITE_EOP - flush caches, send int */
1397         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1398         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1399         radeon_ring_write(ring, lower_32_bits(addr));
1400         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1401         radeon_ring_write(ring, fence->seq);
1402         radeon_ring_write(ring, 0);
1403 }
1404
1405 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1406 {
1407         struct radeon_ring *ring = &rdev->ring[ib->ring];
1408         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1409         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1410                 PACKET3_SH_ACTION_ENA;
1411
1412         /* set to DX10/11 mode */
1413         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1414         radeon_ring_write(ring, 1);
1415
1416         if (ring->rptr_save_reg) {
1417                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1418                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1419                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1420                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1421                 radeon_ring_write(ring, next_rptr);
1422         }
1423
1424         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1425         radeon_ring_write(ring,
1426 #ifdef __BIG_ENDIAN
1427                           (2 << 0) |
1428 #endif
1429                           (ib->gpu_addr & 0xFFFFFFFC));
1430         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1431         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1432
1433         /* flush read cache over gart for this vmid */
1434         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1435         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1436         radeon_ring_write(ring, 0xFFFFFFFF);
1437         radeon_ring_write(ring, 0);
1438         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1439 }
1440
1441 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1442 {
1443         if (enable)
1444                 WREG32(CP_ME_CNTL, 0);
1445         else {
1446                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1447                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1448                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1449                 WREG32(SCRATCH_UMSK, 0);
1450                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1451         }
1452 }
1453
1454 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1455                         struct radeon_ring *ring)
1456 {
1457         u32 rptr;
1458
1459         if (rdev->wb.enabled)
1460                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1461         else {
1462                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1463                         rptr = RREG32(CP_RB0_RPTR);
1464                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1465                         rptr = RREG32(CP_RB1_RPTR);
1466                 else
1467                         rptr = RREG32(CP_RB2_RPTR);
1468         }
1469
1470         return rptr;
1471 }
1472
1473 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1474                         struct radeon_ring *ring)
1475 {
1476         u32 wptr;
1477
1478         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1479                 wptr = RREG32(CP_RB0_WPTR);
1480         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1481                 wptr = RREG32(CP_RB1_WPTR);
1482         else
1483                 wptr = RREG32(CP_RB2_WPTR);
1484
1485         return wptr;
1486 }
1487
1488 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1489                          struct radeon_ring *ring)
1490 {
1491         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1492                 WREG32(CP_RB0_WPTR, ring->wptr);
1493                 (void)RREG32(CP_RB0_WPTR);
1494         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1495                 WREG32(CP_RB1_WPTR, ring->wptr);
1496                 (void)RREG32(CP_RB1_WPTR);
1497         } else {
1498                 WREG32(CP_RB2_WPTR, ring->wptr);
1499                 (void)RREG32(CP_RB2_WPTR);
1500         }
1501 }
1502
1503 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1504 {
1505         const __be32 *fw_data;
1506         int i;
1507
1508         if (!rdev->me_fw || !rdev->pfp_fw)
1509                 return -EINVAL;
1510
1511         cayman_cp_enable(rdev, false);
1512
1513         fw_data = (const __be32 *)rdev->pfp_fw->data;
1514         WREG32(CP_PFP_UCODE_ADDR, 0);
1515         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1516                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1517         WREG32(CP_PFP_UCODE_ADDR, 0);
1518
1519         fw_data = (const __be32 *)rdev->me_fw->data;
1520         WREG32(CP_ME_RAM_WADDR, 0);
1521         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1522                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1523
1524         WREG32(CP_PFP_UCODE_ADDR, 0);
1525         WREG32(CP_ME_RAM_WADDR, 0);
1526         WREG32(CP_ME_RAM_RADDR, 0);
1527         return 0;
1528 }
1529
1530 static int cayman_cp_start(struct radeon_device *rdev)
1531 {
1532         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1533         int r, i;
1534
1535         r = radeon_ring_lock(rdev, ring, 7);
1536         if (r) {
1537                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1538                 return r;
1539         }
1540         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1541         radeon_ring_write(ring, 0x1);
1542         radeon_ring_write(ring, 0x0);
1543         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1544         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1545         radeon_ring_write(ring, 0);
1546         radeon_ring_write(ring, 0);
1547         radeon_ring_unlock_commit(rdev, ring, false);
1548
1549         cayman_cp_enable(rdev, true);
1550
1551         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1552         if (r) {
1553                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1554                 return r;
1555         }
1556
1557         /* setup clear context state */
1558         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1559         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1560
1561         for (i = 0; i < cayman_default_size; i++)
1562                 radeon_ring_write(ring, cayman_default_state[i]);
1563
1564         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1565         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1566
1567         /* set clear context state */
1568         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1569         radeon_ring_write(ring, 0);
1570
1571         /* SQ_VTX_BASE_VTX_LOC */
1572         radeon_ring_write(ring, 0xc0026f00);
1573         radeon_ring_write(ring, 0x00000000);
1574         radeon_ring_write(ring, 0x00000000);
1575         radeon_ring_write(ring, 0x00000000);
1576
1577         /* Clear consts */
1578         radeon_ring_write(ring, 0xc0036f00);
1579         radeon_ring_write(ring, 0x00000bc4);
1580         radeon_ring_write(ring, 0xffffffff);
1581         radeon_ring_write(ring, 0xffffffff);
1582         radeon_ring_write(ring, 0xffffffff);
1583
1584         radeon_ring_write(ring, 0xc0026900);
1585         radeon_ring_write(ring, 0x00000316);
1586         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1587         radeon_ring_write(ring, 0x00000010); /*  */
1588
1589         radeon_ring_unlock_commit(rdev, ring, false);
1590
1591         /* XXX init other rings */
1592
1593         return 0;
1594 }
1595
1596 static void cayman_cp_fini(struct radeon_device *rdev)
1597 {
1598         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1599         cayman_cp_enable(rdev, false);
1600         radeon_ring_fini(rdev, ring);
1601         radeon_scratch_free(rdev, ring->rptr_save_reg);
1602 }
1603
1604 static int cayman_cp_resume(struct radeon_device *rdev)
1605 {
1606         static const int ridx[] = {
1607                 RADEON_RING_TYPE_GFX_INDEX,
1608                 CAYMAN_RING_TYPE_CP1_INDEX,
1609                 CAYMAN_RING_TYPE_CP2_INDEX
1610         };
1611         static const unsigned cp_rb_cntl[] = {
1612                 CP_RB0_CNTL,
1613                 CP_RB1_CNTL,
1614                 CP_RB2_CNTL,
1615         };
1616         static const unsigned cp_rb_rptr_addr[] = {
1617                 CP_RB0_RPTR_ADDR,
1618                 CP_RB1_RPTR_ADDR,
1619                 CP_RB2_RPTR_ADDR
1620         };
1621         static const unsigned cp_rb_rptr_addr_hi[] = {
1622                 CP_RB0_RPTR_ADDR_HI,
1623                 CP_RB1_RPTR_ADDR_HI,
1624                 CP_RB2_RPTR_ADDR_HI
1625         };
1626         static const unsigned cp_rb_base[] = {
1627                 CP_RB0_BASE,
1628                 CP_RB1_BASE,
1629                 CP_RB2_BASE
1630         };
1631         static const unsigned cp_rb_rptr[] = {
1632                 CP_RB0_RPTR,
1633                 CP_RB1_RPTR,
1634                 CP_RB2_RPTR
1635         };
1636         static const unsigned cp_rb_wptr[] = {
1637                 CP_RB0_WPTR,
1638                 CP_RB1_WPTR,
1639                 CP_RB2_WPTR
1640         };
1641         struct radeon_ring *ring;
1642         int i, r;
1643
1644         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1645         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1646                                  SOFT_RESET_PA |
1647                                  SOFT_RESET_SH |
1648                                  SOFT_RESET_VGT |
1649                                  SOFT_RESET_SPI |
1650                                  SOFT_RESET_SX));
1651         RREG32(GRBM_SOFT_RESET);
1652         mdelay(15);
1653         WREG32(GRBM_SOFT_RESET, 0);
1654         RREG32(GRBM_SOFT_RESET);
1655
1656         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1657         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1658
1659         /* Set the write pointer delay */
1660         WREG32(CP_RB_WPTR_DELAY, 0);
1661
1662         WREG32(CP_DEBUG, (1 << 27));
1663
1664         /* set the wb address whether it's enabled or not */
1665         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1666         WREG32(SCRATCH_UMSK, 0xff);
1667
1668         for (i = 0; i < 3; ++i) {
1669                 uint32_t rb_cntl;
1670                 uint64_t addr;
1671
1672                 /* Set ring buffer size */
1673                 ring = &rdev->ring[ridx[i]];
1674                 rb_cntl = order_base_2(ring->ring_size / 8);
1675                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1676 #ifdef __BIG_ENDIAN
1677                 rb_cntl |= BUF_SWAP_32BIT;
1678 #endif
1679                 WREG32(cp_rb_cntl[i], rb_cntl);
1680
1681                 /* set the wb address whether it's enabled or not */
1682                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1683                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1684                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1685         }
1686
1687         /* set the rb base addr, this causes an internal reset of ALL rings */
1688         for (i = 0; i < 3; ++i) {
1689                 ring = &rdev->ring[ridx[i]];
1690                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1691         }
1692
1693         for (i = 0; i < 3; ++i) {
1694                 /* Initialize the ring buffer's read and write pointers */
1695                 ring = &rdev->ring[ridx[i]];
1696                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1697
1698                 ring->wptr = 0;
1699                 WREG32(cp_rb_rptr[i], 0);
1700                 WREG32(cp_rb_wptr[i], ring->wptr);
1701
1702                 mdelay(1);
1703                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1704         }
1705
1706         /* start the rings */
1707         cayman_cp_start(rdev);
1708         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1709         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1710         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1711         /* this only test cp0 */
1712         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1713         if (r) {
1714                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1715                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1716                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1717                 return r;
1718         }
1719
1720         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1721                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1722
1723         return 0;
1724 }
1725
1726 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1727 {
1728         u32 reset_mask = 0;
1729         u32 tmp;
1730
1731         /* GRBM_STATUS */
1732         tmp = RREG32(GRBM_STATUS);
1733         if (tmp & (PA_BUSY | SC_BUSY |
1734                    SH_BUSY | SX_BUSY |
1735                    TA_BUSY | VGT_BUSY |
1736                    DB_BUSY | CB_BUSY |
1737                    GDS_BUSY | SPI_BUSY |
1738                    IA_BUSY | IA_BUSY_NO_DMA))
1739                 reset_mask |= RADEON_RESET_GFX;
1740
1741         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1742                    CP_BUSY | CP_COHERENCY_BUSY))
1743                 reset_mask |= RADEON_RESET_CP;
1744
1745         if (tmp & GRBM_EE_BUSY)
1746                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1747
1748         /* DMA_STATUS_REG 0 */
1749         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1750         if (!(tmp & DMA_IDLE))
1751                 reset_mask |= RADEON_RESET_DMA;
1752
1753         /* DMA_STATUS_REG 1 */
1754         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1755         if (!(tmp & DMA_IDLE))
1756                 reset_mask |= RADEON_RESET_DMA1;
1757
1758         /* SRBM_STATUS2 */
1759         tmp = RREG32(SRBM_STATUS2);
1760         if (tmp & DMA_BUSY)
1761                 reset_mask |= RADEON_RESET_DMA;
1762
1763         if (tmp & DMA1_BUSY)
1764                 reset_mask |= RADEON_RESET_DMA1;
1765
1766         /* SRBM_STATUS */
1767         tmp = RREG32(SRBM_STATUS);
1768         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1769                 reset_mask |= RADEON_RESET_RLC;
1770
1771         if (tmp & IH_BUSY)
1772                 reset_mask |= RADEON_RESET_IH;
1773
1774         if (tmp & SEM_BUSY)
1775                 reset_mask |= RADEON_RESET_SEM;
1776
1777         if (tmp & GRBM_RQ_PENDING)
1778                 reset_mask |= RADEON_RESET_GRBM;
1779
1780         if (tmp & VMC_BUSY)
1781                 reset_mask |= RADEON_RESET_VMC;
1782
1783         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1784                    MCC_BUSY | MCD_BUSY))
1785                 reset_mask |= RADEON_RESET_MC;
1786
1787         if (evergreen_is_display_hung(rdev))
1788                 reset_mask |= RADEON_RESET_DISPLAY;
1789
1790         /* VM_L2_STATUS */
1791         tmp = RREG32(VM_L2_STATUS);
1792         if (tmp & L2_BUSY)
1793                 reset_mask |= RADEON_RESET_VMC;
1794
1795         /* Skip MC reset as it's mostly likely not hung, just busy */
1796         if (reset_mask & RADEON_RESET_MC) {
1797                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1798                 reset_mask &= ~RADEON_RESET_MC;
1799         }
1800
1801         return reset_mask;
1802 }
1803
1804 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1805 {
1806         struct evergreen_mc_save save;
1807         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1808         u32 tmp;
1809
1810         if (reset_mask == 0)
1811                 return;
1812
1813         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1814
1815         evergreen_print_gpu_status_regs(rdev);
1816         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1817                  RREG32(0x14F8));
1818         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1819                  RREG32(0x14D8));
1820         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1821                  RREG32(0x14FC));
1822         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1823                  RREG32(0x14DC));
1824
1825         /* Disable CP parsing/prefetching */
1826         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1827
1828         if (reset_mask & RADEON_RESET_DMA) {
1829                 /* dma0 */
1830                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1831                 tmp &= ~DMA_RB_ENABLE;
1832                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1833         }
1834
1835         if (reset_mask & RADEON_RESET_DMA1) {
1836                 /* dma1 */
1837                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1838                 tmp &= ~DMA_RB_ENABLE;
1839                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1840         }
1841
1842         udelay(50);
1843
1844         evergreen_mc_stop(rdev, &save);
1845         if (evergreen_mc_wait_for_idle(rdev)) {
1846                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1847         }
1848
1849         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1850                 grbm_soft_reset = SOFT_RESET_CB |
1851                         SOFT_RESET_DB |
1852                         SOFT_RESET_GDS |
1853                         SOFT_RESET_PA |
1854                         SOFT_RESET_SC |
1855                         SOFT_RESET_SPI |
1856                         SOFT_RESET_SH |
1857                         SOFT_RESET_SX |
1858                         SOFT_RESET_TC |
1859                         SOFT_RESET_TA |
1860                         SOFT_RESET_VGT |
1861                         SOFT_RESET_IA;
1862         }
1863
1864         if (reset_mask & RADEON_RESET_CP) {
1865                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1866
1867                 srbm_soft_reset |= SOFT_RESET_GRBM;
1868         }
1869
1870         if (reset_mask & RADEON_RESET_DMA)
1871                 srbm_soft_reset |= SOFT_RESET_DMA;
1872
1873         if (reset_mask & RADEON_RESET_DMA1)
1874                 srbm_soft_reset |= SOFT_RESET_DMA1;
1875
1876         if (reset_mask & RADEON_RESET_DISPLAY)
1877                 srbm_soft_reset |= SOFT_RESET_DC;
1878
1879         if (reset_mask & RADEON_RESET_RLC)
1880                 srbm_soft_reset |= SOFT_RESET_RLC;
1881
1882         if (reset_mask & RADEON_RESET_SEM)
1883                 srbm_soft_reset |= SOFT_RESET_SEM;
1884
1885         if (reset_mask & RADEON_RESET_IH)
1886                 srbm_soft_reset |= SOFT_RESET_IH;
1887
1888         if (reset_mask & RADEON_RESET_GRBM)
1889                 srbm_soft_reset |= SOFT_RESET_GRBM;
1890
1891         if (reset_mask & RADEON_RESET_VMC)
1892                 srbm_soft_reset |= SOFT_RESET_VMC;
1893
1894         if (!(rdev->flags & RADEON_IS_IGP)) {
1895                 if (reset_mask & RADEON_RESET_MC)
1896                         srbm_soft_reset |= SOFT_RESET_MC;
1897         }
1898
1899         if (grbm_soft_reset) {
1900                 tmp = RREG32(GRBM_SOFT_RESET);
1901                 tmp |= grbm_soft_reset;
1902                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1903                 WREG32(GRBM_SOFT_RESET, tmp);
1904                 tmp = RREG32(GRBM_SOFT_RESET);
1905
1906                 udelay(50);
1907
1908                 tmp &= ~grbm_soft_reset;
1909                 WREG32(GRBM_SOFT_RESET, tmp);
1910                 tmp = RREG32(GRBM_SOFT_RESET);
1911         }
1912
1913         if (srbm_soft_reset) {
1914                 tmp = RREG32(SRBM_SOFT_RESET);
1915                 tmp |= srbm_soft_reset;
1916                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1917                 WREG32(SRBM_SOFT_RESET, tmp);
1918                 tmp = RREG32(SRBM_SOFT_RESET);
1919
1920                 udelay(50);
1921
1922                 tmp &= ~srbm_soft_reset;
1923                 WREG32(SRBM_SOFT_RESET, tmp);
1924                 tmp = RREG32(SRBM_SOFT_RESET);
1925         }
1926
1927         /* Wait a little for things to settle down */
1928         udelay(50);
1929
1930         evergreen_mc_resume(rdev, &save);
1931         udelay(50);
1932
1933         evergreen_print_gpu_status_regs(rdev);
1934 }
1935
1936 int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1937 {
1938         u32 reset_mask;
1939
1940         if (hard) {
1941                 evergreen_gpu_pci_config_reset(rdev);
1942                 return 0;
1943         }
1944
1945         reset_mask = cayman_gpu_check_soft_reset(rdev);
1946
1947         if (reset_mask)
1948                 r600_set_bios_scratch_engine_hung(rdev, true);
1949
1950         cayman_gpu_soft_reset(rdev, reset_mask);
1951
1952         reset_mask = cayman_gpu_check_soft_reset(rdev);
1953
1954         if (reset_mask)
1955                 evergreen_gpu_pci_config_reset(rdev);
1956
1957         r600_set_bios_scratch_engine_hung(rdev, false);
1958
1959         return 0;
1960 }
1961
1962 /**
1963  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1964  *
1965  * @rdev: radeon_device pointer
1966  * @ring: radeon_ring structure holding ring information
1967  *
1968  * Check if the GFX engine is locked up.
1969  * Returns true if the engine appears to be locked up, false if not.
1970  */
1971 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1972 {
1973         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1974
1975         if (!(reset_mask & (RADEON_RESET_GFX |
1976                             RADEON_RESET_COMPUTE |
1977                             RADEON_RESET_CP))) {
1978                 radeon_ring_lockup_update(rdev, ring);
1979                 return false;
1980         }
1981         return radeon_ring_test_lockup(rdev, ring);
1982 }
1983
1984 static void cayman_uvd_init(struct radeon_device *rdev)
1985 {
1986         int r;
1987
1988         if (!rdev->has_uvd)
1989                 return;
1990
1991         r = radeon_uvd_init(rdev);
1992         if (r) {
1993                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1994                 /*
1995                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1996                  * to early fails uvd_v2_2_resume() and thus nothing happens
1997                  * there. So it is pointless to try to go through that code
1998                  * hence why we disable uvd here.
1999                  */
2000                 rdev->has_uvd = false;
2001                 return;
2002         }
2003         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
2004         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
2005 }
2006
2007 static void cayman_uvd_start(struct radeon_device *rdev)
2008 {
2009         int r;
2010
2011         if (!rdev->has_uvd)
2012                 return;
2013
2014         r = uvd_v2_2_resume(rdev);
2015         if (r) {
2016                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2017                 goto error;
2018         }
2019         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2020         if (r) {
2021                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2022                 goto error;
2023         }
2024         return;
2025
2026 error:
2027         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2028 }
2029
2030 static void cayman_uvd_resume(struct radeon_device *rdev)
2031 {
2032         struct radeon_ring *ring;
2033         int r;
2034
2035         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2036                 return;
2037
2038         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2039         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2040         if (r) {
2041                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2042                 return;
2043         }
2044         r = uvd_v1_0_init(rdev);
2045         if (r) {
2046                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2047                 return;
2048         }
2049 }
2050
2051 static void cayman_vce_init(struct radeon_device *rdev)
2052 {
2053         int r;
2054
2055         /* Only set for CHIP_ARUBA */
2056         if (!rdev->has_vce)
2057                 return;
2058
2059         r = radeon_vce_init(rdev);
2060         if (r) {
2061                 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2062                 /*
2063                  * At this point rdev->vce.vcpu_bo is NULL which trickles down
2064                  * to early fails cayman_vce_start() and thus nothing happens
2065                  * there. So it is pointless to try to go through that code
2066                  * hence why we disable vce here.
2067                  */
2068                 rdev->has_vce = false;
2069                 return;
2070         }
2071         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2072         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2073         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2074         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2075 }
2076
2077 static void cayman_vce_start(struct radeon_device *rdev)
2078 {
2079         int r;
2080
2081         if (!rdev->has_vce)
2082                 return;
2083
2084         r = radeon_vce_resume(rdev);
2085         if (r) {
2086                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2087                 goto error;
2088         }
2089         r = vce_v1_0_resume(rdev);
2090         if (r) {
2091                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2092                 goto error;
2093         }
2094         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2095         if (r) {
2096                 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2097                 goto error;
2098         }
2099         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2100         if (r) {
2101                 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2102                 goto error;
2103         }
2104         return;
2105
2106 error:
2107         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2108         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2109 }
2110
2111 static void cayman_vce_resume(struct radeon_device *rdev)
2112 {
2113         struct radeon_ring *ring;
2114         int r;
2115
2116         if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2117                 return;
2118
2119         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2120         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2121         if (r) {
2122                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2123                 return;
2124         }
2125         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2126         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2127         if (r) {
2128                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2129                 return;
2130         }
2131         r = vce_v1_0_init(rdev);
2132         if (r) {
2133                 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2134                 return;
2135         }
2136 }
2137
2138 static int cayman_startup(struct radeon_device *rdev)
2139 {
2140         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2141         int r;
2142
2143         /* enable pcie gen2 link */
2144         evergreen_pcie_gen2_enable(rdev);
2145         /* enable aspm */
2146         evergreen_program_aspm(rdev);
2147
2148         /* scratch needs to be initialized before MC */
2149         r = r600_vram_scratch_init(rdev);
2150         if (r)
2151                 return r;
2152
2153         evergreen_mc_program(rdev);
2154
2155         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2156                 r = ni_mc_load_microcode(rdev);
2157                 if (r) {
2158                         DRM_ERROR("Failed to load MC firmware!\n");
2159                         return r;
2160                 }
2161         }
2162
2163         r = cayman_pcie_gart_enable(rdev);
2164         if (r)
2165                 return r;
2166         cayman_gpu_init(rdev);
2167
2168         /* allocate rlc buffers */
2169         if (rdev->flags & RADEON_IS_IGP) {
2170                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2171                 rdev->rlc.reg_list_size =
2172                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2173                 rdev->rlc.cs_data = cayman_cs_data;
2174                 r = sumo_rlc_init(rdev);
2175                 if (r) {
2176                         DRM_ERROR("Failed to init rlc BOs!\n");
2177                         return r;
2178                 }
2179         }
2180
2181         /* allocate wb buffer */
2182         r = radeon_wb_init(rdev);
2183         if (r)
2184                 return r;
2185
2186         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2187         if (r) {
2188                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2189                 return r;
2190         }
2191
2192         cayman_uvd_start(rdev);
2193         cayman_vce_start(rdev);
2194
2195         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2196         if (r) {
2197                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2198                 return r;
2199         }
2200
2201         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2202         if (r) {
2203                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2204                 return r;
2205         }
2206
2207         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2208         if (r) {
2209                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2210                 return r;
2211         }
2212
2213         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2214         if (r) {
2215                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2216                 return r;
2217         }
2218
2219         /* Enable IRQ */
2220         if (!rdev->irq.installed) {
2221                 r = radeon_irq_kms_init(rdev);
2222                 if (r)
2223                         return r;
2224         }
2225
2226         r = r600_irq_init(rdev);
2227         if (r) {
2228                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2229                 radeon_irq_kms_fini(rdev);
2230                 return r;
2231         }
2232         evergreen_irq_set(rdev);
2233
2234         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2235                              RADEON_CP_PACKET2);
2236         if (r)
2237                 return r;
2238
2239         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2240         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2241                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2242         if (r)
2243                 return r;
2244
2245         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2246         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2247                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2248         if (r)
2249                 return r;
2250
2251         r = cayman_cp_load_microcode(rdev);
2252         if (r)
2253                 return r;
2254         r = cayman_cp_resume(rdev);
2255         if (r)
2256                 return r;
2257
2258         r = cayman_dma_resume(rdev);
2259         if (r)
2260                 return r;
2261
2262         cayman_uvd_resume(rdev);
2263         cayman_vce_resume(rdev);
2264
2265         r = radeon_ib_pool_init(rdev);
2266         if (r) {
2267                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2268                 return r;
2269         }
2270
2271         r = radeon_vm_manager_init(rdev);
2272         if (r) {
2273                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2274                 return r;
2275         }
2276
2277         r = radeon_audio_init(rdev);
2278         if (r)
2279                 return r;
2280
2281         return 0;
2282 }
2283
2284 int cayman_resume(struct radeon_device *rdev)
2285 {
2286         int r;
2287
2288         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2289          * posting will perform necessary task to bring back GPU into good
2290          * shape.
2291          */
2292         /* post card */
2293         atom_asic_init(rdev->mode_info.atom_context);
2294
2295         /* init golden registers */
2296         ni_init_golden_registers(rdev);
2297
2298         if (rdev->pm.pm_method == PM_METHOD_DPM)
2299                 radeon_pm_resume(rdev);
2300
2301         rdev->accel_working = true;
2302         r = cayman_startup(rdev);
2303         if (r) {
2304                 DRM_ERROR("cayman startup failed on resume\n");
2305                 rdev->accel_working = false;
2306                 return r;
2307         }
2308         return r;
2309 }
2310
2311 int cayman_suspend(struct radeon_device *rdev)
2312 {
2313         radeon_pm_suspend(rdev);
2314         radeon_audio_fini(rdev);
2315         radeon_vm_manager_fini(rdev);
2316         cayman_cp_enable(rdev, false);
2317         cayman_dma_stop(rdev);
2318         if (rdev->has_uvd) {
2319                 uvd_v1_0_fini(rdev);
2320                 radeon_uvd_suspend(rdev);
2321         }
2322         evergreen_irq_suspend(rdev);
2323         radeon_wb_disable(rdev);
2324         cayman_pcie_gart_disable(rdev);
2325         return 0;
2326 }
2327
2328 /* Plan is to move initialization in that function and use
2329  * helper function so that radeon_device_init pretty much
2330  * do nothing more than calling asic specific function. This
2331  * should also allow to remove a bunch of callback function
2332  * like vram_info.
2333  */
2334 int cayman_init(struct radeon_device *rdev)
2335 {
2336         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2337         int r;
2338
2339         /* Read BIOS */
2340         if (!radeon_get_bios(rdev)) {
2341                 if (ASIC_IS_AVIVO(rdev))
2342                         return -EINVAL;
2343         }
2344         /* Must be an ATOMBIOS */
2345         if (!rdev->is_atom_bios) {
2346                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2347                 return -EINVAL;
2348         }
2349         r = radeon_atombios_init(rdev);
2350         if (r)
2351                 return r;
2352
2353         /* Post card if necessary */
2354         if (!radeon_card_posted(rdev)) {
2355                 if (!rdev->bios) {
2356                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2357                         return -EINVAL;
2358                 }
2359                 DRM_INFO("GPU not posted. posting now...\n");
2360                 atom_asic_init(rdev->mode_info.atom_context);
2361         }
2362         /* init golden registers */
2363         ni_init_golden_registers(rdev);
2364         /* Initialize scratch registers */
2365         r600_scratch_init(rdev);
2366         /* Initialize surface registers */
2367         radeon_surface_init(rdev);
2368         /* Initialize clocks */
2369         radeon_get_clock_info(rdev->ddev);
2370         /* Fence driver */
2371         r = radeon_fence_driver_init(rdev);
2372         if (r)
2373                 return r;
2374         /* initialize memory controller */
2375         r = evergreen_mc_init(rdev);
2376         if (r)
2377                 return r;
2378         /* Memory manager */
2379         r = radeon_bo_init(rdev);
2380         if (r)
2381                 return r;
2382
2383         if (rdev->flags & RADEON_IS_IGP) {
2384                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2385                         r = ni_init_microcode(rdev);
2386                         if (r) {
2387                                 DRM_ERROR("Failed to load firmware!\n");
2388                                 /*(DEBLOBBED)*/
2389                         }
2390                 }
2391         } else {
2392                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2393                         r = ni_init_microcode(rdev);
2394                         if (r) {
2395                                 DRM_ERROR("Failed to load firmware!\n");
2396                                 /*(DEBLOBBED)*/
2397                         }
2398                 }
2399         }
2400
2401         /* Initialize power management */
2402         radeon_pm_init(rdev);
2403
2404         ring->ring_obj = NULL;
2405         r600_ring_init(rdev, ring, 1024 * 1024);
2406
2407         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2408         ring->ring_obj = NULL;
2409         r600_ring_init(rdev, ring, 64 * 1024);
2410
2411         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2412         ring->ring_obj = NULL;
2413         r600_ring_init(rdev, ring, 64 * 1024);
2414
2415         cayman_uvd_init(rdev);
2416         cayman_vce_init(rdev);
2417
2418         rdev->ih.ring_obj = NULL;
2419         r600_ih_ring_init(rdev, 64 * 1024);
2420
2421         r = r600_pcie_gart_init(rdev);
2422         if (r)
2423                 return r;
2424
2425         rdev->accel_working = true;
2426         r = cayman_startup(rdev);
2427         if (r) {
2428                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2429                 cayman_cp_fini(rdev);
2430                 cayman_dma_fini(rdev);
2431                 r600_irq_fini(rdev);
2432                 if (rdev->flags & RADEON_IS_IGP)
2433                         sumo_rlc_fini(rdev);
2434                 radeon_wb_fini(rdev);
2435                 radeon_ib_pool_fini(rdev);
2436                 radeon_vm_manager_fini(rdev);
2437                 radeon_irq_kms_fini(rdev);
2438                 cayman_pcie_gart_fini(rdev);
2439                 rdev->accel_working = false;
2440         }
2441
2442         /* Don't start up if the MC ucode is missing.
2443          * The default clocks and voltages before the MC ucode
2444          * is loaded are not suffient for advanced operations.
2445          *
2446          * We can skip this check for TN, because there is no MC
2447          * ucode.
2448          */
2449         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2450                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2451                 return -EINVAL;
2452         }
2453
2454         return 0;
2455 }
2456
2457 void cayman_fini(struct radeon_device *rdev)
2458 {
2459         radeon_pm_fini(rdev);
2460         cayman_cp_fini(rdev);
2461         cayman_dma_fini(rdev);
2462         r600_irq_fini(rdev);
2463         if (rdev->flags & RADEON_IS_IGP)
2464                 sumo_rlc_fini(rdev);
2465         radeon_wb_fini(rdev);
2466         radeon_vm_manager_fini(rdev);
2467         radeon_ib_pool_fini(rdev);
2468         radeon_irq_kms_fini(rdev);
2469         uvd_v1_0_fini(rdev);
2470         radeon_uvd_fini(rdev);
2471         if (rdev->has_vce)
2472                 radeon_vce_fini(rdev);
2473         cayman_pcie_gart_fini(rdev);
2474         r600_vram_scratch_fini(rdev);
2475         radeon_gem_fini(rdev);
2476         radeon_fence_driver_fini(rdev);
2477         radeon_bo_fini(rdev);
2478         radeon_atombios_fini(rdev);
2479         kfree(rdev->bios);
2480         rdev->bios = NULL;
2481 }
2482
2483 /*
2484  * vm
2485  */
2486 int cayman_vm_init(struct radeon_device *rdev)
2487 {
2488         /* number of VMs */
2489         rdev->vm_manager.nvm = 8;
2490         /* base offset of vram pages */
2491         if (rdev->flags & RADEON_IS_IGP) {
2492                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2493                 tmp <<= 22;
2494                 rdev->vm_manager.vram_base_offset = tmp;
2495         } else
2496                 rdev->vm_manager.vram_base_offset = 0;
2497         return 0;
2498 }
2499
2500 void cayman_vm_fini(struct radeon_device *rdev)
2501 {
2502 }
2503
2504 /**
2505  * cayman_vm_decode_fault - print human readable fault info
2506  *
2507  * @rdev: radeon_device pointer
2508  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2509  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2510  *
2511  * Print human readable fault information (cayman/TN).
2512  */
2513 void cayman_vm_decode_fault(struct radeon_device *rdev,
2514                             u32 status, u32 addr)
2515 {
2516         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2517         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2518         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2519         char *block;
2520
2521         switch (mc_id) {
2522         case 32:
2523         case 16:
2524         case 96:
2525         case 80:
2526         case 160:
2527         case 144:
2528         case 224:
2529         case 208:
2530                 block = "CB";
2531                 break;
2532         case 33:
2533         case 17:
2534         case 97:
2535         case 81:
2536         case 161:
2537         case 145:
2538         case 225:
2539         case 209:
2540                 block = "CB_FMASK";
2541                 break;
2542         case 34:
2543         case 18:
2544         case 98:
2545         case 82:
2546         case 162:
2547         case 146:
2548         case 226:
2549         case 210:
2550                 block = "CB_CMASK";
2551                 break;
2552         case 35:
2553         case 19:
2554         case 99:
2555         case 83:
2556         case 163:
2557         case 147:
2558         case 227:
2559         case 211:
2560                 block = "CB_IMMED";
2561                 break;
2562         case 36:
2563         case 20:
2564         case 100:
2565         case 84:
2566         case 164:
2567         case 148:
2568         case 228:
2569         case 212:
2570                 block = "DB";
2571                 break;
2572         case 37:
2573         case 21:
2574         case 101:
2575         case 85:
2576         case 165:
2577         case 149:
2578         case 229:
2579         case 213:
2580                 block = "DB_HTILE";
2581                 break;
2582         case 38:
2583         case 22:
2584         case 102:
2585         case 86:
2586         case 166:
2587         case 150:
2588         case 230:
2589         case 214:
2590                 block = "SX";
2591                 break;
2592         case 39:
2593         case 23:
2594         case 103:
2595         case 87:
2596         case 167:
2597         case 151:
2598         case 231:
2599         case 215:
2600                 block = "DB_STEN";
2601                 break;
2602         case 40:
2603         case 24:
2604         case 104:
2605         case 88:
2606         case 232:
2607         case 216:
2608         case 168:
2609         case 152:
2610                 block = "TC_TFETCH";
2611                 break;
2612         case 41:
2613         case 25:
2614         case 105:
2615         case 89:
2616         case 233:
2617         case 217:
2618         case 169:
2619         case 153:
2620                 block = "TC_VFETCH";
2621                 break;
2622         case 42:
2623         case 26:
2624         case 106:
2625         case 90:
2626         case 234:
2627         case 218:
2628         case 170:
2629         case 154:
2630                 block = "VC";
2631                 break;
2632         case 112:
2633                 block = "CP";
2634                 break;
2635         case 113:
2636         case 114:
2637                 block = "SH";
2638                 break;
2639         case 115:
2640                 block = "VGT";
2641                 break;
2642         case 178:
2643                 block = "IH";
2644                 break;
2645         case 51:
2646                 block = "RLC";
2647                 break;
2648         case 55:
2649                 block = "DMA";
2650                 break;
2651         case 56:
2652                 block = "HDP";
2653                 break;
2654         default:
2655                 block = "unknown";
2656                 break;
2657         }
2658
2659         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2660                protections, vmid, addr,
2661                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2662                block, mc_id);
2663 }
2664
2665 /**
2666  * cayman_vm_flush - vm flush using the CP
2667  *
2668  * @rdev: radeon_device pointer
2669  *
2670  * Update the page table base and flush the VM TLB
2671  * using the CP (cayman-si).
2672  */
2673 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2674                      unsigned vm_id, uint64_t pd_addr)
2675 {
2676         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2677         radeon_ring_write(ring, pd_addr >> 12);
2678
2679         /* flush hdp cache */
2680         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2681         radeon_ring_write(ring, 0x1);
2682
2683         /* bits 0-7 are the VM contexts0-7 */
2684         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2685         radeon_ring_write(ring, 1 << vm_id);
2686
2687         /* wait for the invalidate to complete */
2688         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2689         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2690                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2691         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2692         radeon_ring_write(ring, 0);
2693         radeon_ring_write(ring, 0); /* ref */
2694         radeon_ring_write(ring, 0); /* mask */
2695         radeon_ring_write(ring, 0x20); /* poll interval */
2696
2697         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2698         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2699         radeon_ring_write(ring, 0x0);
2700 }
2701
2702 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2703 {
2704         struct atom_clock_dividers dividers;
2705         int r, i;
2706
2707         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2708                                            ecclk, false, &dividers);
2709         if (r)
2710                 return r;
2711
2712         for (i = 0; i < 100; i++) {
2713                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2714                         break;
2715                 mdelay(10);
2716         }
2717         if (i == 100)
2718                 return -ETIMEDOUT;
2719
2720         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2721
2722         for (i = 0; i < 100; i++) {
2723                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2724                         break;
2725                 mdelay(10);
2726         }
2727         if (i == 100)
2728                 return -ETIMEDOUT;
2729
2730         return 0;
2731 }