drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/pci.h>
  27 #include <linux/slab.h>
  28
  29 #include <drm/drm_vblank.h>
  30 #include <drm/radeon_drm.h>
  31 #include <drm/drm_fourcc.h>
  32
  33 #include "atom.h"
  34 #include "avivod.h"
  35 #include "cik.h"
  36 #include "ni.h"
  37 #include "rv770.h"
  38 #include "evergreen.h"
  39 #include "evergreen_blit_shaders.h"
  40 #include "evergreen_reg.h"
  41 #include "evergreend.h"
  42 #include "radeon.h"
  43 #include "radeon_asic.h"
  44 #include "radeon_audio.h"
  45 #include "radeon_ucode.h"
  46 #include "si.h"
  47
  48 #define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  49 #define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  50 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  51
  52 /*
  53  * Indirect registers accessor
  54  */
  55 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  56 {
  57         unsigned long flags;
  58         u32 r;
  59
  60         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  61         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  62         r = RREG32(EVERGREEN_CG_IND_DATA);
  63         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  64         return r;
  65 }
  66
  67 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  68 {
  69         unsigned long flags;
  70
  71         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  72         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  73         WREG32(EVERGREEN_CG_IND_DATA, (v));
  74         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  75 }
  76
  77 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  78 {
  79         unsigned long flags;
  80         u32 r;
  81
  82         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  83         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  84         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  85         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  86         return r;
  87 }
  88
  89 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  90 {
  91         unsigned long flags;
  92
  93         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  94         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  95         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  96         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  97 }
  98
  99 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
 100 {
 101         unsigned long flags;
 102         u32 r;
 103
 104         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 105         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 106         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 107         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 108         return r;
 109 }
 110
 111 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 112 {
 113         unsigned long flags;
 114
 115         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 116         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 117         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 118         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 119 }
 120
 121 static const u32 crtc_offsets[6] =
 122 {
 123         EVERGREEN_CRTC0_REGISTER_OFFSET,
 124         EVERGREEN_CRTC1_REGISTER_OFFSET,
 125         EVERGREEN_CRTC2_REGISTER_OFFSET,
 126         EVERGREEN_CRTC3_REGISTER_OFFSET,
 127         EVERGREEN_CRTC4_REGISTER_OFFSET,
 128         EVERGREEN_CRTC5_REGISTER_OFFSET
 129 };
 130
 131 #include "clearstate_evergreen.h"
 132
 133 static const u32 sumo_rlc_save_restore_register_list[] =
 134 {
 135         0x98fc,
 136         0x9830,
 137         0x9834,
 138         0x9838,
 139         0x9870,
 140         0x9874,
 141         0x8a14,
 142         0x8b24,
 143         0x8bcc,
 144         0x8b10,
 145         0x8d00,
 146         0x8d04,
 147         0x8c00,
 148         0x8c04,
 149         0x8c08,
 150         0x8c0c,
 151         0x8d8c,
 152         0x8c20,
 153         0x8c24,
 154         0x8c28,
 155         0x8c18,
 156         0x8c1c,
 157         0x8cf0,
 158         0x8e2c,
 159         0x8e38,
 160         0x8c30,
 161         0x9508,
 162         0x9688,
 163         0x9608,
 164         0x960c,
 165         0x9610,
 166         0x9614,
 167         0x88c4,
 168         0x88d4,
 169         0xa008,
 170         0x900c,
 171         0x9100,
 172         0x913c,
 173         0x98f8,
 174         0x98f4,
 175         0x9b7c,
 176         0x3f8c,
 177         0x8950,
 178         0x8954,
 179         0x8a18,
 180         0x8b28,
 181         0x9144,
 182         0x9148,
 183         0x914c,
 184         0x3f90,
 185         0x3f94,
 186         0x915c,
 187         0x9160,
 188         0x9178,
 189         0x917c,
 190         0x9180,
 191         0x918c,
 192         0x9190,
 193         0x9194,
 194         0x9198,
 195         0x919c,
 196         0x91a8,
 197         0x91ac,
 198         0x91b0,
 199         0x91b4,
 200         0x91b8,
 201         0x91c4,
 202         0x91c8,
 203         0x91cc,
 204         0x91d0,
 205         0x91d4,
 206         0x91e0,
 207         0x91e4,
 208         0x91ec,
 209         0x91f0,
 210         0x91f4,
 211         0x9200,
 212         0x9204,
 213         0x929c,
 214         0x9150,
 215         0x802c,
 216 };
 217
 218 static void evergreen_gpu_init(struct radeon_device *rdev);
 219 void evergreen_fini(struct radeon_device *rdev);
 220 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 221 void evergreen_program_aspm(struct radeon_device *rdev);
 222
 223 static const u32 evergreen_golden_registers[] =
 224 {
 225         0x3f90, 0xffff0000, 0xff000000,
 226         0x9148, 0xffff0000, 0xff000000,
 227         0x3f94, 0xffff0000, 0xff000000,
 228         0x914c, 0xffff0000, 0xff000000,
 229         0x9b7c, 0xffffffff, 0x00000000,
 230         0x8a14, 0xffffffff, 0x00000007,
 231         0x8b10, 0xffffffff, 0x00000000,
 232         0x960c, 0xffffffff, 0x54763210,
 233         0x88c4, 0xffffffff, 0x000000c2,
 234         0x88d4, 0xffffffff, 0x00000010,
 235         0x8974, 0xffffffff, 0x00000000,
 236         0xc78, 0x00000080, 0x00000080,
 237         0x5eb4, 0xffffffff, 0x00000002,
 238         0x5e78, 0xffffffff, 0x001000f0,
 239         0x6104, 0x01000300, 0x00000000,
 240         0x5bc0, 0x00300000, 0x00000000,
 241         0x7030, 0xffffffff, 0x00000011,
 242         0x7c30, 0xffffffff, 0x00000011,
 243         0x10830, 0xffffffff, 0x00000011,
 244         0x11430, 0xffffffff, 0x00000011,
 245         0x12030, 0xffffffff, 0x00000011,
 246         0x12c30, 0xffffffff, 0x00000011,
 247         0xd02c, 0xffffffff, 0x08421000,
 248         0x240c, 0xffffffff, 0x00000380,
 249         0x8b24, 0xffffffff, 0x00ff0fff,
 250         0x28a4c, 0x06000000, 0x06000000,
 251         0x10c, 0x00000001, 0x00000001,
 252         0x8d00, 0xffffffff, 0x100e4848,
 253         0x8d04, 0xffffffff, 0x00164745,
 254         0x8c00, 0xffffffff, 0xe4000003,
 255         0x8c04, 0xffffffff, 0x40600060,
 256         0x8c08, 0xffffffff, 0x001c001c,
 257         0x8cf0, 0xffffffff, 0x08e00620,
 258         0x8c20, 0xffffffff, 0x00800080,
 259         0x8c24, 0xffffffff, 0x00800080,
 260         0x8c18, 0xffffffff, 0x20202078,
 261         0x8c1c, 0xffffffff, 0x00001010,
 262         0x28350, 0xffffffff, 0x00000000,
 263         0xa008, 0xffffffff, 0x00010000,
 264         0x5c4, 0xffffffff, 0x00000001,
 265         0x9508, 0xffffffff, 0x00000002,
 266         0x913c, 0x0000000f, 0x0000000a
 267 };
 268
 269 static const u32 evergreen_golden_registers2[] =
 270 {
 271         0x2f4c, 0xffffffff, 0x00000000,
 272         0x54f4, 0xffffffff, 0x00000000,
 273         0x54f0, 0xffffffff, 0x00000000,
 274         0x5498, 0xffffffff, 0x00000000,
 275         0x549c, 0xffffffff, 0x00000000,
 276         0x5494, 0xffffffff, 0x00000000,
 277         0x53cc, 0xffffffff, 0x00000000,
 278         0x53c8, 0xffffffff, 0x00000000,
 279         0x53c4, 0xffffffff, 0x00000000,
 280         0x53c0, 0xffffffff, 0x00000000,
 281         0x53bc, 0xffffffff, 0x00000000,
 282         0x53b8, 0xffffffff, 0x00000000,
 283         0x53b4, 0xffffffff, 0x00000000,
 284         0x53b0, 0xffffffff, 0x00000000
 285 };
 286
 287 static const u32 cypress_mgcg_init[] =
 288 {
 289         0x802c, 0xffffffff, 0xc0000000,
 290         0x5448, 0xffffffff, 0x00000100,
 291         0x55e4, 0xffffffff, 0x00000100,
 292         0x160c, 0xffffffff, 0x00000100,
 293         0x5644, 0xffffffff, 0x00000100,
 294         0xc164, 0xffffffff, 0x00000100,
 295         0x8a18, 0xffffffff, 0x00000100,
 296         0x897c, 0xffffffff, 0x06000100,
 297         0x8b28, 0xffffffff, 0x00000100,
 298         0x9144, 0xffffffff, 0x00000100,
 299         0x9a60, 0xffffffff, 0x00000100,
 300         0x9868, 0xffffffff, 0x00000100,
 301         0x8d58, 0xffffffff, 0x00000100,
 302         0x9510, 0xffffffff, 0x00000100,
 303         0x949c, 0xffffffff, 0x00000100,
 304         0x9654, 0xffffffff, 0x00000100,
 305         0x9030, 0xffffffff, 0x00000100,
 306         0x9034, 0xffffffff, 0x00000100,
 307         0x9038, 0xffffffff, 0x00000100,
 308         0x903c, 0xffffffff, 0x00000100,
 309         0x9040, 0xffffffff, 0x00000100,
 310         0xa200, 0xffffffff, 0x00000100,
 311         0xa204, 0xffffffff, 0x00000100,
 312         0xa208, 0xffffffff, 0x00000100,
 313         0xa20c, 0xffffffff, 0x00000100,
 314         0x971c, 0xffffffff, 0x00000100,
 315         0x977c, 0xffffffff, 0x00000100,
 316         0x3f80, 0xffffffff, 0x00000100,
 317         0xa210, 0xffffffff, 0x00000100,
 318         0xa214, 0xffffffff, 0x00000100,
 319         0x4d8, 0xffffffff, 0x00000100,
 320         0x9784, 0xffffffff, 0x00000100,
 321         0x9698, 0xffffffff, 0x00000100,
 322         0x4d4, 0xffffffff, 0x00000200,
 323         0x30cc, 0xffffffff, 0x00000100,
 324         0xd0c0, 0xffffffff, 0xff000100,
 325         0x802c, 0xffffffff, 0x40000000,
 326         0x915c, 0xffffffff, 0x00010000,
 327         0x9160, 0xffffffff, 0x00030002,
 328         0x9178, 0xffffffff, 0x00070000,
 329         0x917c, 0xffffffff, 0x00030002,
 330         0x9180, 0xffffffff, 0x00050004,
 331         0x918c, 0xffffffff, 0x00010006,
 332         0x9190, 0xffffffff, 0x00090008,
 333         0x9194, 0xffffffff, 0x00070000,
 334         0x9198, 0xffffffff, 0x00030002,
 335         0x919c, 0xffffffff, 0x00050004,
 336         0x91a8, 0xffffffff, 0x00010006,
 337         0x91ac, 0xffffffff, 0x00090008,
 338         0x91b0, 0xffffffff, 0x00070000,
 339         0x91b4, 0xffffffff, 0x00030002,
 340         0x91b8, 0xffffffff, 0x00050004,
 341         0x91c4, 0xffffffff, 0x00010006,
 342         0x91c8, 0xffffffff, 0x00090008,
 343         0x91cc, 0xffffffff, 0x00070000,
 344         0x91d0, 0xffffffff, 0x00030002,
 345         0x91d4, 0xffffffff, 0x00050004,
 346         0x91e0, 0xffffffff, 0x00010006,
 347         0x91e4, 0xffffffff, 0x00090008,
 348         0x91e8, 0xffffffff, 0x00000000,
 349         0x91ec, 0xffffffff, 0x00070000,
 350         0x91f0, 0xffffffff, 0x00030002,
 351         0x91f4, 0xffffffff, 0x00050004,
 352         0x9200, 0xffffffff, 0x00010006,
 353         0x9204, 0xffffffff, 0x00090008,
 354         0x9208, 0xffffffff, 0x00070000,
 355         0x920c, 0xffffffff, 0x00030002,
 356         0x9210, 0xffffffff, 0x00050004,
 357         0x921c, 0xffffffff, 0x00010006,
 358         0x9220, 0xffffffff, 0x00090008,
 359         0x9224, 0xffffffff, 0x00070000,
 360         0x9228, 0xffffffff, 0x00030002,
 361         0x922c, 0xffffffff, 0x00050004,
 362         0x9238, 0xffffffff, 0x00010006,
 363         0x923c, 0xffffffff, 0x00090008,
 364         0x9240, 0xffffffff, 0x00070000,
 365         0x9244, 0xffffffff, 0x00030002,
 366         0x9248, 0xffffffff, 0x00050004,
 367         0x9254, 0xffffffff, 0x00010006,
 368         0x9258, 0xffffffff, 0x00090008,
 369         0x925c, 0xffffffff, 0x00070000,
 370         0x9260, 0xffffffff, 0x00030002,
 371         0x9264, 0xffffffff, 0x00050004,
 372         0x9270, 0xffffffff, 0x00010006,
 373         0x9274, 0xffffffff, 0x00090008,
 374         0x9278, 0xffffffff, 0x00070000,
 375         0x927c, 0xffffffff, 0x00030002,
 376         0x9280, 0xffffffff, 0x00050004,
 377         0x928c, 0xffffffff, 0x00010006,
 378         0x9290, 0xffffffff, 0x00090008,
 379         0x9294, 0xffffffff, 0x00000000,
 380         0x929c, 0xffffffff, 0x00000001,
 381         0x802c, 0xffffffff, 0x40010000,
 382         0x915c, 0xffffffff, 0x00010000,
 383         0x9160, 0xffffffff, 0x00030002,
 384         0x9178, 0xffffffff, 0x00070000,
 385         0x917c, 0xffffffff, 0x00030002,
 386         0x9180, 0xffffffff, 0x00050004,
 387         0x918c, 0xffffffff, 0x00010006,
 388         0x9190, 0xffffffff, 0x00090008,
 389         0x9194, 0xffffffff, 0x00070000,
 390         0x9198, 0xffffffff, 0x00030002,
 391         0x919c, 0xffffffff, 0x00050004,
 392         0x91a8, 0xffffffff, 0x00010006,
 393         0x91ac, 0xffffffff, 0x00090008,
 394         0x91b0, 0xffffffff, 0x00070000,
 395         0x91b4, 0xffffffff, 0x00030002,
 396         0x91b8, 0xffffffff, 0x00050004,
 397         0x91c4, 0xffffffff, 0x00010006,
 398         0x91c8, 0xffffffff, 0x00090008,
 399         0x91cc, 0xffffffff, 0x00070000,
 400         0x91d0, 0xffffffff, 0x00030002,
 401         0x91d4, 0xffffffff, 0x00050004,
 402         0x91e0, 0xffffffff, 0x00010006,
 403         0x91e4, 0xffffffff, 0x00090008,
 404         0x91e8, 0xffffffff, 0x00000000,
 405         0x91ec, 0xffffffff, 0x00070000,
 406         0x91f0, 0xffffffff, 0x00030002,
 407         0x91f4, 0xffffffff, 0x00050004,
 408         0x9200, 0xffffffff, 0x00010006,
 409         0x9204, 0xffffffff, 0x00090008,
 410         0x9208, 0xffffffff, 0x00070000,
 411         0x920c, 0xffffffff, 0x00030002,
 412         0x9210, 0xffffffff, 0x00050004,
 413         0x921c, 0xffffffff, 0x00010006,
 414         0x9220, 0xffffffff, 0x00090008,
 415         0x9224, 0xffffffff, 0x00070000,
 416         0x9228, 0xffffffff, 0x00030002,
 417         0x922c, 0xffffffff, 0x00050004,
 418         0x9238, 0xffffffff, 0x00010006,
 419         0x923c, 0xffffffff, 0x00090008,
 420         0x9240, 0xffffffff, 0x00070000,
 421         0x9244, 0xffffffff, 0x00030002,
 422         0x9248, 0xffffffff, 0x00050004,
 423         0x9254, 0xffffffff, 0x00010006,
 424         0x9258, 0xffffffff, 0x00090008,
 425         0x925c, 0xffffffff, 0x00070000,
 426         0x9260, 0xffffffff, 0x00030002,
 427         0x9264, 0xffffffff, 0x00050004,
 428         0x9270, 0xffffffff, 0x00010006,
 429         0x9274, 0xffffffff, 0x00090008,
 430         0x9278, 0xffffffff, 0x00070000,
 431         0x927c, 0xffffffff, 0x00030002,
 432         0x9280, 0xffffffff, 0x00050004,
 433         0x928c, 0xffffffff, 0x00010006,
 434         0x9290, 0xffffffff, 0x00090008,
 435         0x9294, 0xffffffff, 0x00000000,
 436         0x929c, 0xffffffff, 0x00000001,
 437         0x802c, 0xffffffff, 0xc0000000
 438 };
 439
 440 static const u32 redwood_mgcg_init[] =
 441 {
 442         0x802c, 0xffffffff, 0xc0000000,
 443         0x5448, 0xffffffff, 0x00000100,
 444         0x55e4, 0xffffffff, 0x00000100,
 445         0x160c, 0xffffffff, 0x00000100,
 446         0x5644, 0xffffffff, 0x00000100,
 447         0xc164, 0xffffffff, 0x00000100,
 448         0x8a18, 0xffffffff, 0x00000100,
 449         0x897c, 0xffffffff, 0x06000100,
 450         0x8b28, 0xffffffff, 0x00000100,
 451         0x9144, 0xffffffff, 0x00000100,
 452         0x9a60, 0xffffffff, 0x00000100,
 453         0x9868, 0xffffffff, 0x00000100,
 454         0x8d58, 0xffffffff, 0x00000100,
 455         0x9510, 0xffffffff, 0x00000100,
 456         0x949c, 0xffffffff, 0x00000100,
 457         0x9654, 0xffffffff, 0x00000100,
 458         0x9030, 0xffffffff, 0x00000100,
 459         0x9034, 0xffffffff, 0x00000100,
 460         0x9038, 0xffffffff, 0x00000100,
 461         0x903c, 0xffffffff, 0x00000100,
 462         0x9040, 0xffffffff, 0x00000100,
 463         0xa200, 0xffffffff, 0x00000100,
 464         0xa204, 0xffffffff, 0x00000100,
 465         0xa208, 0xffffffff, 0x00000100,
 466         0xa20c, 0xffffffff, 0x00000100,
 467         0x971c, 0xffffffff, 0x00000100,
 468         0x977c, 0xffffffff, 0x00000100,
 469         0x3f80, 0xffffffff, 0x00000100,
 470         0xa210, 0xffffffff, 0x00000100,
 471         0xa214, 0xffffffff, 0x00000100,
 472         0x4d8, 0xffffffff, 0x00000100,
 473         0x9784, 0xffffffff, 0x00000100,
 474         0x9698, 0xffffffff, 0x00000100,
 475         0x4d4, 0xffffffff, 0x00000200,
 476         0x30cc, 0xffffffff, 0x00000100,
 477         0xd0c0, 0xffffffff, 0xff000100,
 478         0x802c, 0xffffffff, 0x40000000,
 479         0x915c, 0xffffffff, 0x00010000,
 480         0x9160, 0xffffffff, 0x00030002,
 481         0x9178, 0xffffffff, 0x00070000,
 482         0x917c, 0xffffffff, 0x00030002,
 483         0x9180, 0xffffffff, 0x00050004,
 484         0x918c, 0xffffffff, 0x00010006,
 485         0x9190, 0xffffffff, 0x00090008,
 486         0x9194, 0xffffffff, 0x00070000,
 487         0x9198, 0xffffffff, 0x00030002,
 488         0x919c, 0xffffffff, 0x00050004,
 489         0x91a8, 0xffffffff, 0x00010006,
 490         0x91ac, 0xffffffff, 0x00090008,
 491         0x91b0, 0xffffffff, 0x00070000,
 492         0x91b4, 0xffffffff, 0x00030002,
 493         0x91b8, 0xffffffff, 0x00050004,
 494         0x91c4, 0xffffffff, 0x00010006,
 495         0x91c8, 0xffffffff, 0x00090008,
 496         0x91cc, 0xffffffff, 0x00070000,
 497         0x91d0, 0xffffffff, 0x00030002,
 498         0x91d4, 0xffffffff, 0x00050004,
 499         0x91e0, 0xffffffff, 0x00010006,
 500         0x91e4, 0xffffffff, 0x00090008,
 501         0x91e8, 0xffffffff, 0x00000000,
 502         0x91ec, 0xffffffff, 0x00070000,
 503         0x91f0, 0xffffffff, 0x00030002,
 504         0x91f4, 0xffffffff, 0x00050004,
 505         0x9200, 0xffffffff, 0x00010006,
 506         0x9204, 0xffffffff, 0x00090008,
 507         0x9294, 0xffffffff, 0x00000000,
 508         0x929c, 0xffffffff, 0x00000001,
 509         0x802c, 0xffffffff, 0xc0000000
 510 };
 511
 512 static const u32 cedar_golden_registers[] =
 513 {
 514         0x3f90, 0xffff0000, 0xff000000,
 515         0x9148, 0xffff0000, 0xff000000,
 516         0x3f94, 0xffff0000, 0xff000000,
 517         0x914c, 0xffff0000, 0xff000000,
 518         0x9b7c, 0xffffffff, 0x00000000,
 519         0x8a14, 0xffffffff, 0x00000007,
 520         0x8b10, 0xffffffff, 0x00000000,
 521         0x960c, 0xffffffff, 0x54763210,
 522         0x88c4, 0xffffffff, 0x000000c2,
 523         0x88d4, 0xffffffff, 0x00000000,
 524         0x8974, 0xffffffff, 0x00000000,
 525         0xc78, 0x00000080, 0x00000080,
 526         0x5eb4, 0xffffffff, 0x00000002,
 527         0x5e78, 0xffffffff, 0x001000f0,
 528         0x6104, 0x01000300, 0x00000000,
 529         0x5bc0, 0x00300000, 0x00000000,
 530         0x7030, 0xffffffff, 0x00000011,
 531         0x7c30, 0xffffffff, 0x00000011,
 532         0x10830, 0xffffffff, 0x00000011,
 533         0x11430, 0xffffffff, 0x00000011,
 534         0xd02c, 0xffffffff, 0x08421000,
 535         0x240c, 0xffffffff, 0x00000380,
 536         0x8b24, 0xffffffff, 0x00ff0fff,
 537         0x28a4c, 0x06000000, 0x06000000,
 538         0x10c, 0x00000001, 0x00000001,
 539         0x8d00, 0xffffffff, 0x100e4848,
 540         0x8d04, 0xffffffff, 0x00164745,
 541         0x8c00, 0xffffffff, 0xe4000003,
 542         0x8c04, 0xffffffff, 0x40600060,
 543         0x8c08, 0xffffffff, 0x001c001c,
 544         0x8cf0, 0xffffffff, 0x08e00410,
 545         0x8c20, 0xffffffff, 0x00800080,
 546         0x8c24, 0xffffffff, 0x00800080,
 547         0x8c18, 0xffffffff, 0x20202078,
 548         0x8c1c, 0xffffffff, 0x00001010,
 549         0x28350, 0xffffffff, 0x00000000,
 550         0xa008, 0xffffffff, 0x00010000,
 551         0x5c4, 0xffffffff, 0x00000001,
 552         0x9508, 0xffffffff, 0x00000002
 553 };
 554
 555 static const u32 cedar_mgcg_init[] =
 556 {
 557         0x802c, 0xffffffff, 0xc0000000,
 558         0x5448, 0xffffffff, 0x00000100,
 559         0x55e4, 0xffffffff, 0x00000100,
 560         0x160c, 0xffffffff, 0x00000100,
 561         0x5644, 0xffffffff, 0x00000100,
 562         0xc164, 0xffffffff, 0x00000100,
 563         0x8a18, 0xffffffff, 0x00000100,
 564         0x897c, 0xffffffff, 0x06000100,
 565         0x8b28, 0xffffffff, 0x00000100,
 566         0x9144, 0xffffffff, 0x00000100,
 567         0x9a60, 0xffffffff, 0x00000100,
 568         0x9868, 0xffffffff, 0x00000100,
 569         0x8d58, 0xffffffff, 0x00000100,
 570         0x9510, 0xffffffff, 0x00000100,
 571         0x949c, 0xffffffff, 0x00000100,
 572         0x9654, 0xffffffff, 0x00000100,
 573         0x9030, 0xffffffff, 0x00000100,
 574         0x9034, 0xffffffff, 0x00000100,
 575         0x9038, 0xffffffff, 0x00000100,
 576         0x903c, 0xffffffff, 0x00000100,
 577         0x9040, 0xffffffff, 0x00000100,
 578         0xa200, 0xffffffff, 0x00000100,
 579         0xa204, 0xffffffff, 0x00000100,
 580         0xa208, 0xffffffff, 0x00000100,
 581         0xa20c, 0xffffffff, 0x00000100,
 582         0x971c, 0xffffffff, 0x00000100,
 583         0x977c, 0xffffffff, 0x00000100,
 584         0x3f80, 0xffffffff, 0x00000100,
 585         0xa210, 0xffffffff, 0x00000100,
 586         0xa214, 0xffffffff, 0x00000100,
 587         0x4d8, 0xffffffff, 0x00000100,
 588         0x9784, 0xffffffff, 0x00000100,
 589         0x9698, 0xffffffff, 0x00000100,
 590         0x4d4, 0xffffffff, 0x00000200,
 591         0x30cc, 0xffffffff, 0x00000100,
 592         0xd0c0, 0xffffffff, 0xff000100,
 593         0x802c, 0xffffffff, 0x40000000,
 594         0x915c, 0xffffffff, 0x00010000,
 595         0x9178, 0xffffffff, 0x00050000,
 596         0x917c, 0xffffffff, 0x00030002,
 597         0x918c, 0xffffffff, 0x00010004,
 598         0x9190, 0xffffffff, 0x00070006,
 599         0x9194, 0xffffffff, 0x00050000,
 600         0x9198, 0xffffffff, 0x00030002,
 601         0x91a8, 0xffffffff, 0x00010004,
 602         0x91ac, 0xffffffff, 0x00070006,
 603         0x91e8, 0xffffffff, 0x00000000,
 604         0x9294, 0xffffffff, 0x00000000,
 605         0x929c, 0xffffffff, 0x00000001,
 606         0x802c, 0xffffffff, 0xc0000000
 607 };
 608
 609 static const u32 juniper_mgcg_init[] =
 610 {
 611         0x802c, 0xffffffff, 0xc0000000,
 612         0x5448, 0xffffffff, 0x00000100,
 613         0x55e4, 0xffffffff, 0x00000100,
 614         0x160c, 0xffffffff, 0x00000100,
 615         0x5644, 0xffffffff, 0x00000100,
 616         0xc164, 0xffffffff, 0x00000100,
 617         0x8a18, 0xffffffff, 0x00000100,
 618         0x897c, 0xffffffff, 0x06000100,
 619         0x8b28, 0xffffffff, 0x00000100,
 620         0x9144, 0xffffffff, 0x00000100,
 621         0x9a60, 0xffffffff, 0x00000100,
 622         0x9868, 0xffffffff, 0x00000100,
 623         0x8d58, 0xffffffff, 0x00000100,
 624         0x9510, 0xffffffff, 0x00000100,
 625         0x949c, 0xffffffff, 0x00000100,
 626         0x9654, 0xffffffff, 0x00000100,
 627         0x9030, 0xffffffff, 0x00000100,
 628         0x9034, 0xffffffff, 0x00000100,
 629         0x9038, 0xffffffff, 0x00000100,
 630         0x903c, 0xffffffff, 0x00000100,
 631         0x9040, 0xffffffff, 0x00000100,
 632         0xa200, 0xffffffff, 0x00000100,
 633         0xa204, 0xffffffff, 0x00000100,
 634         0xa208, 0xffffffff, 0x00000100,
 635         0xa20c, 0xffffffff, 0x00000100,
 636         0x971c, 0xffffffff, 0x00000100,
 637         0xd0c0, 0xffffffff, 0xff000100,
 638         0x802c, 0xffffffff, 0x40000000,
 639         0x915c, 0xffffffff, 0x00010000,
 640         0x9160, 0xffffffff, 0x00030002,
 641         0x9178, 0xffffffff, 0x00070000,
 642         0x917c, 0xffffffff, 0x00030002,
 643         0x9180, 0xffffffff, 0x00050004,
 644         0x918c, 0xffffffff, 0x00010006,
 645         0x9190, 0xffffffff, 0x00090008,
 646         0x9194, 0xffffffff, 0x00070000,
 647         0x9198, 0xffffffff, 0x00030002,
 648         0x919c, 0xffffffff, 0x00050004,
 649         0x91a8, 0xffffffff, 0x00010006,
 650         0x91ac, 0xffffffff, 0x00090008,
 651         0x91b0, 0xffffffff, 0x00070000,
 652         0x91b4, 0xffffffff, 0x00030002,
 653         0x91b8, 0xffffffff, 0x00050004,
 654         0x91c4, 0xffffffff, 0x00010006,
 655         0x91c8, 0xffffffff, 0x00090008,
 656         0x91cc, 0xffffffff, 0x00070000,
 657         0x91d0, 0xffffffff, 0x00030002,
 658         0x91d4, 0xffffffff, 0x00050004,
 659         0x91e0, 0xffffffff, 0x00010006,
 660         0x91e4, 0xffffffff, 0x00090008,
 661         0x91e8, 0xffffffff, 0x00000000,
 662         0x91ec, 0xffffffff, 0x00070000,
 663         0x91f0, 0xffffffff, 0x00030002,
 664         0x91f4, 0xffffffff, 0x00050004,
 665         0x9200, 0xffffffff, 0x00010006,
 666         0x9204, 0xffffffff, 0x00090008,
 667         0x9208, 0xffffffff, 0x00070000,
 668         0x920c, 0xffffffff, 0x00030002,
 669         0x9210, 0xffffffff, 0x00050004,
 670         0x921c, 0xffffffff, 0x00010006,
 671         0x9220, 0xffffffff, 0x00090008,
 672         0x9224, 0xffffffff, 0x00070000,
 673         0x9228, 0xffffffff, 0x00030002,
 674         0x922c, 0xffffffff, 0x00050004,
 675         0x9238, 0xffffffff, 0x00010006,
 676         0x923c, 0xffffffff, 0x00090008,
 677         0x9240, 0xffffffff, 0x00070000,
 678         0x9244, 0xffffffff, 0x00030002,
 679         0x9248, 0xffffffff, 0x00050004,
 680         0x9254, 0xffffffff, 0x00010006,
 681         0x9258, 0xffffffff, 0x00090008,
 682         0x925c, 0xffffffff, 0x00070000,
 683         0x9260, 0xffffffff, 0x00030002,
 684         0x9264, 0xffffffff, 0x00050004,
 685         0x9270, 0xffffffff, 0x00010006,
 686         0x9274, 0xffffffff, 0x00090008,
 687         0x9278, 0xffffffff, 0x00070000,
 688         0x927c, 0xffffffff, 0x00030002,
 689         0x9280, 0xffffffff, 0x00050004,
 690         0x928c, 0xffffffff, 0x00010006,
 691         0x9290, 0xffffffff, 0x00090008,
 692         0x9294, 0xffffffff, 0x00000000,
 693         0x929c, 0xffffffff, 0x00000001,
 694         0x802c, 0xffffffff, 0xc0000000,
 695         0x977c, 0xffffffff, 0x00000100,
 696         0x3f80, 0xffffffff, 0x00000100,
 697         0xa210, 0xffffffff, 0x00000100,
 698         0xa214, 0xffffffff, 0x00000100,
 699         0x4d8, 0xffffffff, 0x00000100,
 700         0x9784, 0xffffffff, 0x00000100,
 701         0x9698, 0xffffffff, 0x00000100,
 702         0x4d4, 0xffffffff, 0x00000200,
 703         0x30cc, 0xffffffff, 0x00000100,
 704         0x802c, 0xffffffff, 0xc0000000
 705 };
 706
 707 static const u32 supersumo_golden_registers[] =
 708 {
 709         0x5eb4, 0xffffffff, 0x00000002,
 710         0x5c4, 0xffffffff, 0x00000001,
 711         0x7030, 0xffffffff, 0x00000011,
 712         0x7c30, 0xffffffff, 0x00000011,
 713         0x6104, 0x01000300, 0x00000000,
 714         0x5bc0, 0x00300000, 0x00000000,
 715         0x8c04, 0xffffffff, 0x40600060,
 716         0x8c08, 0xffffffff, 0x001c001c,
 717         0x8c20, 0xffffffff, 0x00800080,
 718         0x8c24, 0xffffffff, 0x00800080,
 719         0x8c18, 0xffffffff, 0x20202078,
 720         0x8c1c, 0xffffffff, 0x00001010,
 721         0x918c, 0xffffffff, 0x00010006,
 722         0x91a8, 0xffffffff, 0x00010006,
 723         0x91c4, 0xffffffff, 0x00010006,
 724         0x91e0, 0xffffffff, 0x00010006,
 725         0x9200, 0xffffffff, 0x00010006,
 726         0x9150, 0xffffffff, 0x6e944040,
 727         0x917c, 0xffffffff, 0x00030002,
 728         0x9180, 0xffffffff, 0x00050004,
 729         0x9198, 0xffffffff, 0x00030002,
 730         0x919c, 0xffffffff, 0x00050004,
 731         0x91b4, 0xffffffff, 0x00030002,
 732         0x91b8, 0xffffffff, 0x00050004,
 733         0x91d0, 0xffffffff, 0x00030002,
 734         0x91d4, 0xffffffff, 0x00050004,
 735         0x91f0, 0xffffffff, 0x00030002,
 736         0x91f4, 0xffffffff, 0x00050004,
 737         0x915c, 0xffffffff, 0x00010000,
 738         0x9160, 0xffffffff, 0x00030002,
 739         0x3f90, 0xffff0000, 0xff000000,
 740         0x9178, 0xffffffff, 0x00070000,
 741         0x9194, 0xffffffff, 0x00070000,
 742         0x91b0, 0xffffffff, 0x00070000,
 743         0x91cc, 0xffffffff, 0x00070000,
 744         0x91ec, 0xffffffff, 0x00070000,
 745         0x9148, 0xffff0000, 0xff000000,
 746         0x9190, 0xffffffff, 0x00090008,
 747         0x91ac, 0xffffffff, 0x00090008,
 748         0x91c8, 0xffffffff, 0x00090008,
 749         0x91e4, 0xffffffff, 0x00090008,
 750         0x9204, 0xffffffff, 0x00090008,
 751         0x3f94, 0xffff0000, 0xff000000,
 752         0x914c, 0xffff0000, 0xff000000,
 753         0x929c, 0xffffffff, 0x00000001,
 754         0x8a18, 0xffffffff, 0x00000100,
 755         0x8b28, 0xffffffff, 0x00000100,
 756         0x9144, 0xffffffff, 0x00000100,
 757         0x5644, 0xffffffff, 0x00000100,
 758         0x9b7c, 0xffffffff, 0x00000000,
 759         0x8030, 0xffffffff, 0x0000100a,
 760         0x8a14, 0xffffffff, 0x00000007,
 761         0x8b24, 0xffffffff, 0x00ff0fff,
 762         0x8b10, 0xffffffff, 0x00000000,
 763         0x28a4c, 0x06000000, 0x06000000,
 764         0x4d8, 0xffffffff, 0x00000100,
 765         0x913c, 0xffff000f, 0x0100000a,
 766         0x960c, 0xffffffff, 0x54763210,
 767         0x88c4, 0xffffffff, 0x000000c2,
 768         0x88d4, 0xffffffff, 0x00000010,
 769         0x8974, 0xffffffff, 0x00000000,
 770         0xc78, 0x00000080, 0x00000080,
 771         0x5e78, 0xffffffff, 0x001000f0,
 772         0xd02c, 0xffffffff, 0x08421000,
 773         0xa008, 0xffffffff, 0x00010000,
 774         0x8d00, 0xffffffff, 0x100e4848,
 775         0x8d04, 0xffffffff, 0x00164745,
 776         0x8c00, 0xffffffff, 0xe4000003,
 777         0x8cf0, 0x1fffffff, 0x08e00620,
 778         0x28350, 0xffffffff, 0x00000000,
 779         0x9508, 0xffffffff, 0x00000002
 780 };
 781
 782 static const u32 sumo_golden_registers[] =
 783 {
 784         0x900c, 0x00ffffff, 0x0017071f,
 785         0x8c18, 0xffffffff, 0x10101060,
 786         0x8c1c, 0xffffffff, 0x00001010,
 787         0x8c30, 0x0000000f, 0x00000005,
 788         0x9688, 0x0000000f, 0x00000007
 789 };
 790
 791 static const u32 wrestler_golden_registers[] =
 792 {
 793         0x5eb4, 0xffffffff, 0x00000002,
 794         0x5c4, 0xffffffff, 0x00000001,
 795         0x7030, 0xffffffff, 0x00000011,
 796         0x7c30, 0xffffffff, 0x00000011,
 797         0x6104, 0x01000300, 0x00000000,
 798         0x5bc0, 0x00300000, 0x00000000,
 799         0x918c, 0xffffffff, 0x00010006,
 800         0x91a8, 0xffffffff, 0x00010006,
 801         0x9150, 0xffffffff, 0x6e944040,
 802         0x917c, 0xffffffff, 0x00030002,
 803         0x9198, 0xffffffff, 0x00030002,
 804         0x915c, 0xffffffff, 0x00010000,
 805         0x3f90, 0xffff0000, 0xff000000,
 806         0x9178, 0xffffffff, 0x00070000,
 807         0x9194, 0xffffffff, 0x00070000,
 808         0x9148, 0xffff0000, 0xff000000,
 809         0x9190, 0xffffffff, 0x00090008,
 810         0x91ac, 0xffffffff, 0x00090008,
 811         0x3f94, 0xffff0000, 0xff000000,
 812         0x914c, 0xffff0000, 0xff000000,
 813         0x929c, 0xffffffff, 0x00000001,
 814         0x8a18, 0xffffffff, 0x00000100,
 815         0x8b28, 0xffffffff, 0x00000100,
 816         0x9144, 0xffffffff, 0x00000100,
 817         0x9b7c, 0xffffffff, 0x00000000,
 818         0x8030, 0xffffffff, 0x0000100a,
 819         0x8a14, 0xffffffff, 0x00000001,
 820         0x8b24, 0xffffffff, 0x00ff0fff,
 821         0x8b10, 0xffffffff, 0x00000000,
 822         0x28a4c, 0x06000000, 0x06000000,
 823         0x4d8, 0xffffffff, 0x00000100,
 824         0x913c, 0xffff000f, 0x0100000a,
 825         0x960c, 0xffffffff, 0x54763210,
 826         0x88c4, 0xffffffff, 0x000000c2,
 827         0x88d4, 0xffffffff, 0x00000010,
 828         0x8974, 0xffffffff, 0x00000000,
 829         0xc78, 0x00000080, 0x00000080,
 830         0x5e78, 0xffffffff, 0x001000f0,
 831         0xd02c, 0xffffffff, 0x08421000,
 832         0xa008, 0xffffffff, 0x00010000,
 833         0x8d00, 0xffffffff, 0x100e4848,
 834         0x8d04, 0xffffffff, 0x00164745,
 835         0x8c00, 0xffffffff, 0xe4000003,
 836         0x8cf0, 0x1fffffff, 0x08e00410,
 837         0x28350, 0xffffffff, 0x00000000,
 838         0x9508, 0xffffffff, 0x00000002,
 839         0x900c, 0xffffffff, 0x0017071f,
 840         0x8c18, 0xffffffff, 0x10101060,
 841         0x8c1c, 0xffffffff, 0x00001010
 842 };
 843
 844 static const u32 barts_golden_registers[] =
 845 {
 846         0x5eb4, 0xffffffff, 0x00000002,
 847         0x5e78, 0x8f311ff1, 0x001000f0,
 848         0x3f90, 0xffff0000, 0xff000000,
 849         0x9148, 0xffff0000, 0xff000000,
 850         0x3f94, 0xffff0000, 0xff000000,
 851         0x914c, 0xffff0000, 0xff000000,
 852         0xc78, 0x00000080, 0x00000080,
 853         0xbd4, 0x70073777, 0x00010001,
 854         0xd02c, 0xbfffff1f, 0x08421000,
 855         0xd0b8, 0x03773777, 0x02011003,
 856         0x5bc0, 0x00200000, 0x50100000,
 857         0x98f8, 0x33773777, 0x02011003,
 858         0x98fc, 0xffffffff, 0x76543210,
 859         0x7030, 0x31000311, 0x00000011,
 860         0x2f48, 0x00000007, 0x02011003,
 861         0x6b28, 0x00000010, 0x00000012,
 862         0x7728, 0x00000010, 0x00000012,
 863         0x10328, 0x00000010, 0x00000012,
 864         0x10f28, 0x00000010, 0x00000012,
 865         0x11b28, 0x00000010, 0x00000012,
 866         0x12728, 0x00000010, 0x00000012,
 867         0x240c, 0x000007ff, 0x00000380,
 868         0x8a14, 0xf000001f, 0x00000007,
 869         0x8b24, 0x3fff3fff, 0x00ff0fff,
 870         0x8b10, 0x0000ff0f, 0x00000000,
 871         0x28a4c, 0x07ffffff, 0x06000000,
 872         0x10c, 0x00000001, 0x00010003,
 873         0xa02c, 0xffffffff, 0x0000009b,
 874         0x913c, 0x0000000f, 0x0100000a,
 875         0x8d00, 0xffff7f7f, 0x100e4848,
 876         0x8d04, 0x00ffffff, 0x00164745,
 877         0x8c00, 0xfffc0003, 0xe4000003,
 878         0x8c04, 0xf8ff00ff, 0x40600060,
 879         0x8c08, 0x00ff00ff, 0x001c001c,
 880         0x8cf0, 0x1fff1fff, 0x08e00620,
 881         0x8c20, 0x0fff0fff, 0x00800080,
 882         0x8c24, 0x0fff0fff, 0x00800080,
 883         0x8c18, 0xffffffff, 0x20202078,
 884         0x8c1c, 0x0000ffff, 0x00001010,
 885         0x28350, 0x00000f01, 0x00000000,
 886         0x9508, 0x3700001f, 0x00000002,
 887         0x960c, 0xffffffff, 0x54763210,
 888         0x88c4, 0x001f3ae3, 0x000000c2,
 889         0x88d4, 0x0000001f, 0x00000010,
 890         0x8974, 0xffffffff, 0x00000000
 891 };
 892
 893 static const u32 turks_golden_registers[] =
 894 {
 895         0x5eb4, 0xffffffff, 0x00000002,
 896         0x5e78, 0x8f311ff1, 0x001000f0,
 897         0x8c8, 0x00003000, 0x00001070,
 898         0x8cc, 0x000fffff, 0x00040035,
 899         0x3f90, 0xffff0000, 0xfff00000,
 900         0x9148, 0xffff0000, 0xfff00000,
 901         0x3f94, 0xffff0000, 0xfff00000,
 902         0x914c, 0xffff0000, 0xfff00000,
 903         0xc78, 0x00000080, 0x00000080,
 904         0xbd4, 0x00073007, 0x00010002,
 905         0xd02c, 0xbfffff1f, 0x08421000,
 906         0xd0b8, 0x03773777, 0x02010002,
 907         0x5bc0, 0x00200000, 0x50100000,
 908         0x98f8, 0x33773777, 0x00010002,
 909         0x98fc, 0xffffffff, 0x33221100,
 910         0x7030, 0x31000311, 0x00000011,
 911         0x2f48, 0x33773777, 0x00010002,
 912         0x6b28, 0x00000010, 0x00000012,
 913         0x7728, 0x00000010, 0x00000012,
 914         0x10328, 0x00000010, 0x00000012,
 915         0x10f28, 0x00000010, 0x00000012,
 916         0x11b28, 0x00000010, 0x00000012,
 917         0x12728, 0x00000010, 0x00000012,
 918         0x240c, 0x000007ff, 0x00000380,
 919         0x8a14, 0xf000001f, 0x00000007,
 920         0x8b24, 0x3fff3fff, 0x00ff0fff,
 921         0x8b10, 0x0000ff0f, 0x00000000,
 922         0x28a4c, 0x07ffffff, 0x06000000,
 923         0x10c, 0x00000001, 0x00010003,
 924         0xa02c, 0xffffffff, 0x0000009b,
 925         0x913c, 0x0000000f, 0x0100000a,
 926         0x8d00, 0xffff7f7f, 0x100e4848,
 927         0x8d04, 0x00ffffff, 0x00164745,
 928         0x8c00, 0xfffc0003, 0xe4000003,
 929         0x8c04, 0xf8ff00ff, 0x40600060,
 930         0x8c08, 0x00ff00ff, 0x001c001c,
 931         0x8cf0, 0x1fff1fff, 0x08e00410,
 932         0x8c20, 0x0fff0fff, 0x00800080,
 933         0x8c24, 0x0fff0fff, 0x00800080,
 934         0x8c18, 0xffffffff, 0x20202078,
 935         0x8c1c, 0x0000ffff, 0x00001010,
 936         0x28350, 0x00000f01, 0x00000000,
 937         0x9508, 0x3700001f, 0x00000002,
 938         0x960c, 0xffffffff, 0x54763210,
 939         0x88c4, 0x001f3ae3, 0x000000c2,
 940         0x88d4, 0x0000001f, 0x00000010,
 941         0x8974, 0xffffffff, 0x00000000
 942 };
 943
 944 static const u32 caicos_golden_registers[] =
 945 {
 946         0x5eb4, 0xffffffff, 0x00000002,
 947         0x5e78, 0x8f311ff1, 0x001000f0,
 948         0x8c8, 0x00003420, 0x00001450,
 949         0x8cc, 0x000fffff, 0x00040035,
 950         0x3f90, 0xffff0000, 0xfffc0000,
 951         0x9148, 0xffff0000, 0xfffc0000,
 952         0x3f94, 0xffff0000, 0xfffc0000,
 953         0x914c, 0xffff0000, 0xfffc0000,
 954         0xc78, 0x00000080, 0x00000080,
 955         0xbd4, 0x00073007, 0x00010001,
 956         0xd02c, 0xbfffff1f, 0x08421000,
 957         0xd0b8, 0x03773777, 0x02010001,
 958         0x5bc0, 0x00200000, 0x50100000,
 959         0x98f8, 0x33773777, 0x02010001,
 960         0x98fc, 0xffffffff, 0x33221100,
 961         0x7030, 0x31000311, 0x00000011,
 962         0x2f48, 0x33773777, 0x02010001,
 963         0x6b28, 0x00000010, 0x00000012,
 964         0x7728, 0x00000010, 0x00000012,
 965         0x10328, 0x00000010, 0x00000012,
 966         0x10f28, 0x00000010, 0x00000012,
 967         0x11b28, 0x00000010, 0x00000012,
 968         0x12728, 0x00000010, 0x00000012,
 969         0x240c, 0x000007ff, 0x00000380,
 970         0x8a14, 0xf000001f, 0x00000001,
 971         0x8b24, 0x3fff3fff, 0x00ff0fff,
 972         0x8b10, 0x0000ff0f, 0x00000000,
 973         0x28a4c, 0x07ffffff, 0x06000000,
 974         0x10c, 0x00000001, 0x00010003,
 975         0xa02c, 0xffffffff, 0x0000009b,
 976         0x913c, 0x0000000f, 0x0100000a,
 977         0x8d00, 0xffff7f7f, 0x100e4848,
 978         0x8d04, 0x00ffffff, 0x00164745,
 979         0x8c00, 0xfffc0003, 0xe4000003,
 980         0x8c04, 0xf8ff00ff, 0x40600060,
 981         0x8c08, 0x00ff00ff, 0x001c001c,
 982         0x8cf0, 0x1fff1fff, 0x08e00410,
 983         0x8c20, 0x0fff0fff, 0x00800080,
 984         0x8c24, 0x0fff0fff, 0x00800080,
 985         0x8c18, 0xffffffff, 0x20202078,
 986         0x8c1c, 0x0000ffff, 0x00001010,
 987         0x28350, 0x00000f01, 0x00000000,
 988         0x9508, 0x3700001f, 0x00000002,
 989         0x960c, 0xffffffff, 0x54763210,
 990         0x88c4, 0x001f3ae3, 0x000000c2,
 991         0x88d4, 0x0000001f, 0x00000010,
 992         0x8974, 0xffffffff, 0x00000000
 993 };
 994
 995 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 996 {
 997         switch (rdev->family) {
 998         case CHIP_CYPRESS:
 999         case CHIP_HEMLOCK:
1000                 radeon_program_register_sequence(rdev,
1001                                                  evergreen_golden_registers,
1002                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1003                 radeon_program_register_sequence(rdev,
1004                                                  evergreen_golden_registers2,
1005                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1006                 radeon_program_register_sequence(rdev,
1007                                                  cypress_mgcg_init,
1008                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
1009                 break;
1010         case CHIP_JUNIPER:
1011                 radeon_program_register_sequence(rdev,
1012                                                  evergreen_golden_registers,
1013                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1014                 radeon_program_register_sequence(rdev,
1015                                                  evergreen_golden_registers2,
1016                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1017                 radeon_program_register_sequence(rdev,
1018                                                  juniper_mgcg_init,
1019                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
1020                 break;
1021         case CHIP_REDWOOD:
1022                 radeon_program_register_sequence(rdev,
1023                                                  evergreen_golden_registers,
1024                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1025                 radeon_program_register_sequence(rdev,
1026                                                  evergreen_golden_registers2,
1027                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1028                 radeon_program_register_sequence(rdev,
1029                                                  redwood_mgcg_init,
1030                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
1031                 break;
1032         case CHIP_CEDAR:
1033                 radeon_program_register_sequence(rdev,
1034                                                  cedar_golden_registers,
1035                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
1036                 radeon_program_register_sequence(rdev,
1037                                                  evergreen_golden_registers2,
1038                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1039                 radeon_program_register_sequence(rdev,
1040                                                  cedar_mgcg_init,
1041                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
1042                 break;
1043         case CHIP_PALM:
1044                 radeon_program_register_sequence(rdev,
1045                                                  wrestler_golden_registers,
1046                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
1047                 break;
1048         case CHIP_SUMO:
1049                 radeon_program_register_sequence(rdev,
1050                                                  supersumo_golden_registers,
1051                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1052                 break;
1053         case CHIP_SUMO2:
1054                 radeon_program_register_sequence(rdev,
1055                                                  supersumo_golden_registers,
1056                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1057                 radeon_program_register_sequence(rdev,
1058                                                  sumo_golden_registers,
1059                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
1060                 break;
1061         case CHIP_BARTS:
1062                 radeon_program_register_sequence(rdev,
1063                                                  barts_golden_registers,
1064                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
1065                 break;
1066         case CHIP_TURKS:
1067                 radeon_program_register_sequence(rdev,
1068                                                  turks_golden_registers,
1069                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
1070                 break;
1071         case CHIP_CAICOS:
1072                 radeon_program_register_sequence(rdev,
1073                                                  caicos_golden_registers,
1074                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1075                 break;
1076         default:
1077                 break;
1078         }
1079 }
1080
1081 /**
1082  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1083  *
1084  * @rdev: radeon_device pointer
1085  * @reg: register offset in bytes
1086  * @val: register value
1087  *
1088  * Returns 0 for success or -EINVAL for an invalid register
1089  *
1090  */
1091 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1092                                         u32 reg, u32 *val)
1093 {
1094         switch (reg) {
1095         case GRBM_STATUS:
1096         case GRBM_STATUS_SE0:
1097         case GRBM_STATUS_SE1:
1098         case SRBM_STATUS:
1099         case SRBM_STATUS2:
1100         case DMA_STATUS_REG:
1101         case UVD_STATUS:
1102                 *val = RREG32(reg);
1103                 return 0;
1104         default:
1105                 return -EINVAL;
1106         }
1107 }
1108
1109 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1110                              unsigned *bankh, unsigned *mtaspect,
1111                              unsigned *tile_split)
1112 {
1113         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1114         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1115         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1116         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1117         switch (*bankw) {
1118         default:
1119         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1120         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1121         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1122         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1123         }
1124         switch (*bankh) {
1125         default:
1126         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1127         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1128         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1129         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1130         }
1131         switch (*mtaspect) {
1132         default:
1133         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1134         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1135         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1136         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1137         }
1138 }
1139
1140 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1141                               u32 cntl_reg, u32 status_reg)
1142 {
1143         int r, i;
1144         struct atom_clock_dividers dividers;
1145
1146         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1147                                            clock, false, &dividers);
1148         if (r)
1149                 return r;
1150
1151         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1152
1153         for (i = 0; i < 100; i++) {
1154                 if (RREG32(status_reg) & DCLK_STATUS)
1155                         break;
1156                 mdelay(10);
1157         }
1158         if (i == 100)
1159                 return -ETIMEDOUT;
1160
1161         return 0;
1162 }
1163
1164 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1165 {
1166         int r = 0;
1167         u32 cg_scratch = RREG32(CG_SCRATCH1);
1168
1169         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1170         if (r)
1171                 goto done;
1172         cg_scratch &= 0xffff0000;
1173         cg_scratch |= vclk / 100; /* Mhz */
1174
1175         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1176         if (r)
1177                 goto done;
1178         cg_scratch &= 0x0000ffff;
1179         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1180
1181 done:
1182         WREG32(CG_SCRATCH1, cg_scratch);
1183
1184         return r;
1185 }
1186
1187 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1188 {
1189         /* start off with something large */
1190         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1191         int r;
1192
1193         /* bypass vclk and dclk with bclk */
1194         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1195                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1196                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1197
1198         /* put PLL in bypass mode */
1199         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1200
1201         if (!vclk || !dclk) {
1202                 /* keep the Bypass mode, put PLL to sleep */
1203                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1204                 return 0;
1205         }
1206
1207         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1208                                           16384, 0x03FFFFFF, 0, 128, 5,
1209                                           &fb_div, &vclk_div, &dclk_div);
1210         if (r)
1211                 return r;
1212
1213         /* set VCO_MODE to 1 */
1214         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1215
1216         /* toggle UPLL_SLEEP to 1 then back to 0 */
1217         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1218         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1219
1220         /* deassert UPLL_RESET */
1221         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1222
1223         mdelay(1);
1224
1225         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1226         if (r)
1227                 return r;
1228
1229         /* assert UPLL_RESET again */
1230         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1231
1232         /* disable spread spectrum. */
1233         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1234
1235         /* set feedback divider */
1236         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1237
1238         /* set ref divider to 0 */
1239         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1240
1241         if (fb_div < 307200)
1242                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1243         else
1244                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1245
1246         /* set PDIV_A and PDIV_B */
1247         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1248                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1249                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1250
1251         /* give the PLL some time to settle */
1252         mdelay(15);
1253
1254         /* deassert PLL_RESET */
1255         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1256
1257         mdelay(15);
1258
1259         /* switch from bypass mode to normal mode */
1260         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1261
1262         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1263         if (r)
1264                 return r;
1265
1266         /* switch VCLK and DCLK selection */
1267         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1268                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1269                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1270
1271         mdelay(100);
1272
1273         return 0;
1274 }
1275
1276 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1277 {
1278         int readrq;
1279         u16 v;
1280
1281         readrq = pcie_get_readrq(rdev->pdev);
1282         v = ffs(readrq) - 8;
1283         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1284          * to avoid hangs or perfomance issues
1285          */
1286         if ((v == 0) || (v == 6) || (v == 7))
1287                 pcie_set_readrq(rdev->pdev, 512);
1288 }
1289
1290 void dce4_program_fmt(struct drm_encoder *encoder)
1291 {
1292         struct drm_device *dev = encoder->dev;
1293         struct radeon_device *rdev = dev->dev_private;
1294         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1295         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1296         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1297         int bpc = 0;
1298         u32 tmp = 0;
1299         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1300
1301         if (connector) {
1302                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1303                 bpc = radeon_get_monitor_bpc(connector);
1304                 dither = radeon_connector->dither;
1305         }
1306
1307         /* LVDS/eDP FMT is set up by atom */
1308         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1309                 return;
1310
1311         /* not needed for analog */
1312         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1313             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1314                 return;
1315
1316         if (bpc == 0)
1317                 return;
1318
1319         switch (bpc) {
1320         case 6:
1321                 if (dither == RADEON_FMT_DITHER_ENABLE)
1322                         /* XXX sort out optimal dither settings */
1323                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1324                                 FMT_SPATIAL_DITHER_EN);
1325                 else
1326                         tmp |= FMT_TRUNCATE_EN;
1327                 break;
1328         case 8:
1329                 if (dither == RADEON_FMT_DITHER_ENABLE)
1330                         /* XXX sort out optimal dither settings */
1331                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1332                                 FMT_RGB_RANDOM_ENABLE |
1333                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1334                 else
1335                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1336                 break;
1337         case 10:
1338         default:
1339                 /* not needed */
1340                 break;
1341         }
1342
1343         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1344 }
1345
1346 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1347 {
1348         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1349                 return true;
1350         else
1351                 return false;
1352 }
1353
1354 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1355 {
1356         u32 pos1, pos2;
1357
1358         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1359         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1360
1361         if (pos1 != pos2)
1362                 return true;
1363         else
1364                 return false;
1365 }
1366
1367 /**
1368  * dce4_wait_for_vblank - vblank wait asic callback.
1369  *
1370  * @rdev: radeon_device pointer
1371  * @crtc: crtc to wait for vblank on
1372  *
1373  * Wait for vblank on the requested crtc (evergreen+).
1374  */
1375 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1376 {
1377         unsigned i = 0;
1378
1379         if (crtc >= rdev->num_crtc)
1380                 return;
1381
1382         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1383                 return;
1384
1385         /* depending on when we hit vblank, we may be close to active; if so,
1386          * wait for another frame.
1387          */
1388         while (dce4_is_in_vblank(rdev, crtc)) {
1389                 if (i++ % 100 == 0) {
1390                         if (!dce4_is_counter_moving(rdev, crtc))
1391                                 break;
1392                 }
1393         }
1394
1395         while (!dce4_is_in_vblank(rdev, crtc)) {
1396                 if (i++ % 100 == 0) {
1397                         if (!dce4_is_counter_moving(rdev, crtc))
1398                                 break;
1399                 }
1400         }
1401 }
1402
1403 /**
1404  * evergreen_page_flip - pageflip callback.
1405  *
1406  * @rdev: radeon_device pointer
1407  * @crtc_id: crtc to cleanup pageflip on
1408  * @crtc_base: new address of the crtc (GPU MC address)
1409  * @async: asynchronous flip
1410  *
1411  * Triggers the actual pageflip by updating the primary
1412  * surface base address (evergreen+).
1413  */
1414 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1415                          bool async)
1416 {
1417         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1418         struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1419
1420         /* flip at hsync for async, default is vsync */
1421         WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1422                async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1423         /* update pitch */
1424         WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1425                fb->pitches[0] / fb->format->cpp[0]);
1426         /* update the scanout addresses */
1427         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1428                upper_32_bits(crtc_base));
1429         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1430                (u32)crtc_base);
1431         /* post the write */
1432         RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1433 }
1434
1435 /**
1436  * evergreen_page_flip_pending - check if page flip is still pending
1437  *
1438  * @rdev: radeon_device pointer
1439  * @crtc_id: crtc to check
1440  *
1441  * Returns the current update pending status.
1442  */
1443 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1444 {
1445         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1446
1447         /* Return current update_pending status: */
1448         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1449                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1450 }
1451
1452 /* get temperature in millidegrees */
1453 int evergreen_get_temp(struct radeon_device *rdev)
1454 {
1455         u32 temp, toffset;
1456         int actual_temp = 0;
1457
1458         if (rdev->family == CHIP_JUNIPER) {
1459                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1460                         TOFFSET_SHIFT;
1461                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1462                         TS0_ADC_DOUT_SHIFT;
1463
1464                 if (toffset & 0x100)
1465                         actual_temp = temp / 2 - (0x200 - toffset);
1466                 else
1467                         actual_temp = temp / 2 + toffset;
1468
1469                 actual_temp = actual_temp * 1000;
1470
1471         } else {
1472                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1473                         ASIC_T_SHIFT;
1474
1475                 if (temp & 0x400)
1476                         actual_temp = -256;
1477                 else if (temp & 0x200)
1478                         actual_temp = 255;
1479                 else if (temp & 0x100) {
1480                         actual_temp = temp & 0x1ff;
1481                         actual_temp |= ~0x1ff;
1482                 } else
1483                         actual_temp = temp & 0xff;
1484
1485                 actual_temp = (actual_temp * 1000) / 2;
1486         }
1487
1488         return actual_temp;
1489 }
1490
1491 int sumo_get_temp(struct radeon_device *rdev)
1492 {
1493         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1494         int actual_temp = temp - 49;
1495
1496         return actual_temp * 1000;
1497 }
1498
1499 /**
1500  * sumo_pm_init_profile - Initialize power profiles callback.
1501  *
1502  * @rdev: radeon_device pointer
1503  *
1504  * Initialize the power states used in profile mode
1505  * (sumo, trinity, SI).
1506  * Used for profile mode only.
1507  */
1508 void sumo_pm_init_profile(struct radeon_device *rdev)
1509 {
1510         int idx;
1511
1512         /* default */
1513         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1514         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1515         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1516         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1517
1518         /* low,mid sh/mh */
1519         if (rdev->flags & RADEON_IS_MOBILITY)
1520                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1521         else
1522                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1523
1524         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1525         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1526         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1527         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1528
1529         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1530         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1531         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1532         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1533
1534         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1535         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1536         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1537         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1538
1539         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1540         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1541         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1542         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1543
1544         /* high sh/mh */
1545         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1546         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1547         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1548         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1549         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1550                 rdev->pm.power_state[idx].num_clock_modes - 1;
1551
1552         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1553         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1554         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1555         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1556                 rdev->pm.power_state[idx].num_clock_modes - 1;
1557 }
1558
1559 /**
1560  * btc_pm_init_profile - Initialize power profiles callback.
1561  *
1562  * @rdev: radeon_device pointer
1563  *
1564  * Initialize the power states used in profile mode
1565  * (BTC, cayman).
1566  * Used for profile mode only.
1567  */
1568 void btc_pm_init_profile(struct radeon_device *rdev)
1569 {
1570         int idx;
1571
1572         /* default */
1573         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1574         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1575         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1576         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1577         /* starting with BTC, there is one state that is used for both
1578          * MH and SH.  Difference is that we always use the high clock index for
1579          * mclk.
1580          */
1581         if (rdev->flags & RADEON_IS_MOBILITY)
1582                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1583         else
1584                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1585         /* low sh */
1586         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1587         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1588         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1589         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1590         /* mid sh */
1591         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1592         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1593         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1594         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1595         /* high sh */
1596         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1597         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1598         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1599         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1600         /* low mh */
1601         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1602         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1603         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1604         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1605         /* mid mh */
1606         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1607         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1608         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1609         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1610         /* high mh */
1611         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1612         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1613         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1614         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1615 }
1616
1617 /**
1618  * evergreen_pm_misc - set additional pm hw parameters callback.
1619  *
1620  * @rdev: radeon_device pointer
1621  *
1622  * Set non-clock parameters associated with a power state
1623  * (voltage, etc.) (evergreen+).
1624  */
1625 void evergreen_pm_misc(struct radeon_device *rdev)
1626 {
1627         int req_ps_idx = rdev->pm.requested_power_state_index;
1628         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1629         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1630         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1631
1632         if (voltage->type == VOLTAGE_SW) {
1633                 /* 0xff0x are flags rather then an actual voltage */
1634                 if ((voltage->voltage & 0xff00) == 0xff00)
1635                         return;
1636                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1637                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1638                         rdev->pm.current_vddc = voltage->voltage;
1639                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1640                 }
1641
1642                 /* starting with BTC, there is one state that is used for both
1643                  * MH and SH.  Difference is that we always use the high clock index for
1644                  * mclk and vddci.
1645                  */
1646                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1647                     (rdev->family >= CHIP_BARTS) &&
1648                     rdev->pm.active_crtc_count &&
1649                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1650                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1651                         voltage = &rdev->pm.power_state[req_ps_idx].
1652                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1653
1654                 /* 0xff0x are flags rather then an actual voltage */
1655                 if ((voltage->vddci & 0xff00) == 0xff00)
1656                         return;
1657                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1658                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1659                         rdev->pm.current_vddci = voltage->vddci;
1660                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1661                 }
1662         }
1663 }
1664
1665 /**
1666  * evergreen_pm_prepare - pre-power state change callback.
1667  *
1668  * @rdev: radeon_device pointer
1669  *
1670  * Prepare for a power state change (evergreen+).
1671  */
1672 void evergreen_pm_prepare(struct radeon_device *rdev)
1673 {
1674         struct drm_device *ddev = rdev->ddev;
1675         struct drm_crtc *crtc;
1676         struct radeon_crtc *radeon_crtc;
1677         u32 tmp;
1678
1679         /* disable any active CRTCs */
1680         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1681                 radeon_crtc = to_radeon_crtc(crtc);
1682                 if (radeon_crtc->enabled) {
1683                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1684                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1685                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1686                 }
1687         }
1688 }
1689
1690 /**
1691  * evergreen_pm_finish - post-power state change callback.
1692  *
1693  * @rdev: radeon_device pointer
1694  *
1695  * Clean up after a power state change (evergreen+).
1696  */
1697 void evergreen_pm_finish(struct radeon_device *rdev)
1698 {
1699         struct drm_device *ddev = rdev->ddev;
1700         struct drm_crtc *crtc;
1701         struct radeon_crtc *radeon_crtc;
1702         u32 tmp;
1703
1704         /* enable any active CRTCs */
1705         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1706                 radeon_crtc = to_radeon_crtc(crtc);
1707                 if (radeon_crtc->enabled) {
1708                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1709                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1710                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1711                 }
1712         }
1713 }
1714
1715 /**
1716  * evergreen_hpd_sense - hpd sense callback.
1717  *
1718  * @rdev: radeon_device pointer
1719  * @hpd: hpd (hotplug detect) pin
1720  *
1721  * Checks if a digital monitor is connected (evergreen+).
1722  * Returns true if connected, false if not connected.
1723  */
1724 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1725 {
1726         if (hpd == RADEON_HPD_NONE)
1727                 return false;
1728
1729         return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1730 }
1731
1732 /**
1733  * evergreen_hpd_set_polarity - hpd set polarity callback.
1734  *
1735  * @rdev: radeon_device pointer
1736  * @hpd: hpd (hotplug detect) pin
1737  *
1738  * Set the polarity of the hpd pin (evergreen+).
1739  */
1740 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1741                                 enum radeon_hpd_id hpd)
1742 {
1743         bool connected = evergreen_hpd_sense(rdev, hpd);
1744
1745         if (hpd == RADEON_HPD_NONE)
1746                 return;
1747
1748         if (connected)
1749                 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1750         else
1751                 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1752 }
1753
1754 /**
1755  * evergreen_hpd_init - hpd setup callback.
1756  *
1757  * @rdev: radeon_device pointer
1758  *
1759  * Setup the hpd pins used by the card (evergreen+).
1760  * Enable the pin, set the polarity, and enable the hpd interrupts.
1761  */
1762 void evergreen_hpd_init(struct radeon_device *rdev)
1763 {
1764         struct drm_device *dev = rdev->ddev;
1765         struct drm_connector *connector;
1766         unsigned enabled = 0;
1767         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1768                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1769
1770         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1771                 enum radeon_hpd_id hpd =
1772                         to_radeon_connector(connector)->hpd.hpd;
1773
1774                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1775                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1776                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1777                          * aux dp channel on imac and help (but not completely fix)
1778                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1779                          * also avoid interrupt storms during dpms.
1780                          */
1781                         continue;
1782                 }
1783
1784                 if (hpd == RADEON_HPD_NONE)
1785                         continue;
1786
1787                 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1788                 enabled |= 1 << hpd;
1789
1790                 radeon_hpd_set_polarity(rdev, hpd);
1791         }
1792         radeon_irq_kms_enable_hpd(rdev, enabled);
1793 }
1794
1795 /**
1796  * evergreen_hpd_fini - hpd tear down callback.
1797  *
1798  * @rdev: radeon_device pointer
1799  *
1800  * Tear down the hpd pins used by the card (evergreen+).
1801  * Disable the hpd interrupts.
1802  */
1803 void evergreen_hpd_fini(struct radeon_device *rdev)
1804 {
1805         struct drm_device *dev = rdev->ddev;
1806         struct drm_connector *connector;
1807         unsigned disabled = 0;
1808
1809         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1810                 enum radeon_hpd_id hpd =
1811                         to_radeon_connector(connector)->hpd.hpd;
1812
1813                 if (hpd == RADEON_HPD_NONE)
1814                         continue;
1815
1816                 WREG32(DC_HPDx_CONTROL(hpd), 0);
1817                 disabled |= 1 << hpd;
1818         }
1819         radeon_irq_kms_disable_hpd(rdev, disabled);
1820 }
1821
1822 /* watermark setup */
1823
1824 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1825                                         struct radeon_crtc *radeon_crtc,
1826                                         struct drm_display_mode *mode,
1827                                         struct drm_display_mode *other_mode)
1828 {
1829         u32 tmp, buffer_alloc, i;
1830         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1831         /*
1832          * Line Buffer Setup
1833          * There are 3 line buffers, each one shared by 2 display controllers.
1834          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1835          * the display controllers.  The paritioning is done via one of four
1836          * preset allocations specified in bits 2:0:
1837          * first display controller
1838          *  0 - first half of lb (3840 * 2)
1839          *  1 - first 3/4 of lb (5760 * 2)
1840          *  2 - whole lb (7680 * 2), other crtc must be disabled
1841          *  3 - first 1/4 of lb (1920 * 2)
1842          * second display controller
1843          *  4 - second half of lb (3840 * 2)
1844          *  5 - second 3/4 of lb (5760 * 2)
1845          *  6 - whole lb (7680 * 2), other crtc must be disabled
1846          *  7 - last 1/4 of lb (1920 * 2)
1847          */
1848         /* this can get tricky if we have two large displays on a paired group
1849          * of crtcs.  Ideally for multiple large displays we'd assign them to
1850          * non-linked crtcs for maximum line buffer allocation.
1851          */
1852         if (radeon_crtc->base.enabled && mode) {
1853                 if (other_mode) {
1854                         tmp = 0; /* 1/2 */
1855                         buffer_alloc = 1;
1856                 } else {
1857                         tmp = 2; /* whole */
1858                         buffer_alloc = 2;
1859                 }
1860         } else {
1861                 tmp = 0;
1862                 buffer_alloc = 0;
1863         }
1864
1865         /* second controller of the pair uses second half of the lb */
1866         if (radeon_crtc->crtc_id % 2)
1867                 tmp += 4;
1868         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1869
1870         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1871                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1872                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1873                 for (i = 0; i < rdev->usec_timeout; i++) {
1874                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1875                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1876                                 break;
1877                         udelay(1);
1878                 }
1879         }
1880
1881         if (radeon_crtc->base.enabled && mode) {
1882                 switch (tmp) {
1883                 case 0:
1884                 case 4:
1885                 default:
1886                         if (ASIC_IS_DCE5(rdev))
1887                                 return 4096 * 2;
1888                         else
1889                                 return 3840 * 2;
1890                 case 1:
1891                 case 5:
1892                         if (ASIC_IS_DCE5(rdev))
1893                                 return 6144 * 2;
1894                         else
1895                                 return 5760 * 2;
1896                 case 2:
1897                 case 6:
1898                         if (ASIC_IS_DCE5(rdev))
1899                                 return 8192 * 2;
1900                         else
1901                                 return 7680 * 2;
1902                 case 3:
1903                 case 7:
1904                         if (ASIC_IS_DCE5(rdev))
1905                                 return 2048 * 2;
1906                         else
1907                                 return 1920 * 2;
1908                 }
1909         }
1910
1911         /* controller not enabled, so no lb used */
1912         return 0;
1913 }
1914
1915 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1916 {
1917         u32 tmp = RREG32(MC_SHARED_CHMAP);
1918
1919         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1920         case 0:
1921         default:
1922                 return 1;
1923         case 1:
1924                 return 2;
1925         case 2:
1926                 return 4;
1927         case 3:
1928                 return 8;
1929         }
1930 }
1931
1932 struct evergreen_wm_params {
1933         u32 dram_channels; /* number of dram channels */
1934         u32 yclk;          /* bandwidth per dram data pin in kHz */
1935         u32 sclk;          /* engine clock in kHz */
1936         u32 disp_clk;      /* display clock in kHz */
1937         u32 src_width;     /* viewport width */
1938         u32 active_time;   /* active display time in ns */
1939         u32 blank_time;    /* blank time in ns */
1940         bool interlaced;    /* mode is interlaced */
1941         fixed20_12 vsc;    /* vertical scale ratio */
1942         u32 num_heads;     /* number of active crtcs */
1943         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1944         u32 lb_size;       /* line buffer allocated to pipe */
1945         u32 vtaps;         /* vertical scaler taps */
1946 };
1947
1948 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1949 {
1950         /* Calculate DRAM Bandwidth and the part allocated to display. */
1951         fixed20_12 dram_efficiency; /* 0.7 */
1952         fixed20_12 yclk, dram_channels, bandwidth;
1953         fixed20_12 a;
1954
1955         a.full = dfixed_const(1000);
1956         yclk.full = dfixed_const(wm->yclk);
1957         yclk.full = dfixed_div(yclk, a);
1958         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1959         a.full = dfixed_const(10);
1960         dram_efficiency.full = dfixed_const(7);
1961         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1962         bandwidth.full = dfixed_mul(dram_channels, yclk);
1963         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1964
1965         return dfixed_trunc(bandwidth);
1966 }
1967
1968 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1969 {
1970         /* Calculate DRAM Bandwidth and the part allocated to display. */
1971         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1972         fixed20_12 yclk, dram_channels, bandwidth;
1973         fixed20_12 a;
1974
1975         a.full = dfixed_const(1000);
1976         yclk.full = dfixed_const(wm->yclk);
1977         yclk.full = dfixed_div(yclk, a);
1978         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1979         a.full = dfixed_const(10);
1980         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1981         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1982         bandwidth.full = dfixed_mul(dram_channels, yclk);
1983         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1984
1985         return dfixed_trunc(bandwidth);
1986 }
1987
1988 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1989 {
1990         /* Calculate the display Data return Bandwidth */
1991         fixed20_12 return_efficiency; /* 0.8 */
1992         fixed20_12 sclk, bandwidth;
1993         fixed20_12 a;
1994
1995         a.full = dfixed_const(1000);
1996         sclk.full = dfixed_const(wm->sclk);
1997         sclk.full = dfixed_div(sclk, a);
1998         a.full = dfixed_const(10);
1999         return_efficiency.full = dfixed_const(8);
2000         return_efficiency.full = dfixed_div(return_efficiency, a);
2001         a.full = dfixed_const(32);
2002         bandwidth.full = dfixed_mul(a, sclk);
2003         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2004
2005         return dfixed_trunc(bandwidth);
2006 }
2007
2008 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2009 {
2010         /* Calculate the DMIF Request Bandwidth */
2011         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2012         fixed20_12 disp_clk, bandwidth;
2013         fixed20_12 a;
2014
2015         a.full = dfixed_const(1000);
2016         disp_clk.full = dfixed_const(wm->disp_clk);
2017         disp_clk.full = dfixed_div(disp_clk, a);
2018         a.full = dfixed_const(10);
2019         disp_clk_request_efficiency.full = dfixed_const(8);
2020         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2021         a.full = dfixed_const(32);
2022         bandwidth.full = dfixed_mul(a, disp_clk);
2023         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2024
2025         return dfixed_trunc(bandwidth);
2026 }
2027
2028 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2029 {
2030         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2031         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2032         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2033         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2034
2035         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2036 }
2037
2038 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2039 {
2040         /* Calculate the display mode Average Bandwidth
2041          * DisplayMode should contain the source and destination dimensions,
2042          * timing, etc.
2043          */
2044         fixed20_12 bpp;
2045         fixed20_12 line_time;
2046         fixed20_12 src_width;
2047         fixed20_12 bandwidth;
2048         fixed20_12 a;
2049
2050         a.full = dfixed_const(1000);
2051         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2052         line_time.full = dfixed_div(line_time, a);
2053         bpp.full = dfixed_const(wm->bytes_per_pixel);
2054         src_width.full = dfixed_const(wm->src_width);
2055         bandwidth.full = dfixed_mul(src_width, bpp);
2056         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2057         bandwidth.full = dfixed_div(bandwidth, line_time);
2058
2059         return dfixed_trunc(bandwidth);
2060 }
2061
2062 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2063 {
2064         /* First calcualte the latency in ns */
2065         u32 mc_latency = 2000; /* 2000 ns. */
2066         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2067         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2068         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2069         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2070         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2071                 (wm->num_heads * cursor_line_pair_return_time);
2072         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2073         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2074         fixed20_12 a, b, c;
2075
2076         if (wm->num_heads == 0)
2077                 return 0;
2078
2079         a.full = dfixed_const(2);
2080         b.full = dfixed_const(1);
2081         if ((wm->vsc.full > a.full) ||
2082             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2083             (wm->vtaps >= 5) ||
2084             ((wm->vsc.full >= a.full) && wm->interlaced))
2085                 max_src_lines_per_dst_line = 4;
2086         else
2087                 max_src_lines_per_dst_line = 2;
2088
2089         a.full = dfixed_const(available_bandwidth);
2090         b.full = dfixed_const(wm->num_heads);
2091         a.full = dfixed_div(a, b);
2092
2093         lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2094
2095         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2096         b.full = dfixed_const(1000);
2097         c.full = dfixed_const(lb_fill_bw);
2098         b.full = dfixed_div(c, b);
2099         a.full = dfixed_div(a, b);
2100         line_fill_time = dfixed_trunc(a);
2101
2102         if (line_fill_time < wm->active_time)
2103                 return latency;
2104         else
2105                 return latency + (line_fill_time - wm->active_time);
2106
2107 }
2108
2109 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2110 {
2111         if (evergreen_average_bandwidth(wm) <=
2112             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2113                 return true;
2114         else
2115                 return false;
2116 };
2117
2118 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2119 {
2120         if (evergreen_average_bandwidth(wm) <=
2121             (evergreen_available_bandwidth(wm) / wm->num_heads))
2122                 return true;
2123         else
2124                 return false;
2125 };
2126
2127 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2128 {
2129         u32 lb_partitions = wm->lb_size / wm->src_width;
2130         u32 line_time = wm->active_time + wm->blank_time;
2131         u32 latency_tolerant_lines;
2132         u32 latency_hiding;
2133         fixed20_12 a;
2134
2135         a.full = dfixed_const(1);
2136         if (wm->vsc.full > a.full)
2137                 latency_tolerant_lines = 1;
2138         else {
2139                 if (lb_partitions <= (wm->vtaps + 1))
2140                         latency_tolerant_lines = 1;
2141                 else
2142                         latency_tolerant_lines = 2;
2143         }
2144
2145         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2146
2147         if (evergreen_latency_watermark(wm) <= latency_hiding)
2148                 return true;
2149         else
2150                 return false;
2151 }
2152
2153 static void evergreen_program_watermarks(struct radeon_device *rdev,
2154                                          struct radeon_crtc *radeon_crtc,
2155                                          u32 lb_size, u32 num_heads)
2156 {
2157         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2158         struct evergreen_wm_params wm_low, wm_high;
2159         u32 dram_channels;
2160         u32 active_time;
2161         u32 line_time = 0;
2162         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2163         u32 priority_a_mark = 0, priority_b_mark = 0;
2164         u32 priority_a_cnt = PRIORITY_OFF;
2165         u32 priority_b_cnt = PRIORITY_OFF;
2166         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2167         u32 tmp, arb_control3;
2168         fixed20_12 a, b, c;
2169
2170         if (radeon_crtc->base.enabled && num_heads && mode) {
2171                 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2172                                             (u32)mode->clock);
2173                 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2174                                           (u32)mode->clock);
2175                 line_time = min(line_time, (u32)65535);
2176                 priority_a_cnt = 0;
2177                 priority_b_cnt = 0;
2178                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2179
2180                 /* watermark for high clocks */
2181                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2182                         wm_high.yclk =
2183                                 radeon_dpm_get_mclk(rdev, false) * 10;
2184                         wm_high.sclk =
2185                                 radeon_dpm_get_sclk(rdev, false) * 10;
2186                 } else {
2187                         wm_high.yclk = rdev->pm.current_mclk * 10;
2188                         wm_high.sclk = rdev->pm.current_sclk * 10;
2189                 }
2190
2191                 wm_high.disp_clk = mode->clock;
2192                 wm_high.src_width = mode->crtc_hdisplay;
2193                 wm_high.active_time = active_time;
2194                 wm_high.blank_time = line_time - wm_high.active_time;
2195                 wm_high.interlaced = false;
2196                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2197                         wm_high.interlaced = true;
2198                 wm_high.vsc = radeon_crtc->vsc;
2199                 wm_high.vtaps = 1;
2200                 if (radeon_crtc->rmx_type != RMX_OFF)
2201                         wm_high.vtaps = 2;
2202                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2203                 wm_high.lb_size = lb_size;
2204                 wm_high.dram_channels = dram_channels;
2205                 wm_high.num_heads = num_heads;
2206
2207                 /* watermark for low clocks */
2208                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2209                         wm_low.yclk =
2210                                 radeon_dpm_get_mclk(rdev, true) * 10;
2211                         wm_low.sclk =
2212                                 radeon_dpm_get_sclk(rdev, true) * 10;
2213                 } else {
2214                         wm_low.yclk = rdev->pm.current_mclk * 10;
2215                         wm_low.sclk = rdev->pm.current_sclk * 10;
2216                 }
2217
2218                 wm_low.disp_clk = mode->clock;
2219                 wm_low.src_width = mode->crtc_hdisplay;
2220                 wm_low.active_time = active_time;
2221                 wm_low.blank_time = line_time - wm_low.active_time;
2222                 wm_low.interlaced = false;
2223                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2224                         wm_low.interlaced = true;
2225                 wm_low.vsc = radeon_crtc->vsc;
2226                 wm_low.vtaps = 1;
2227                 if (radeon_crtc->rmx_type != RMX_OFF)
2228                         wm_low.vtaps = 2;
2229                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2230                 wm_low.lb_size = lb_size;
2231                 wm_low.dram_channels = dram_channels;
2232                 wm_low.num_heads = num_heads;
2233
2234                 /* set for high clocks */
2235                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2236                 /* set for low clocks */
2237                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2238
2239                 /* possibly force display priority to high */
2240                 /* should really do this at mode validation time... */
2241                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2242                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2243                     !evergreen_check_latency_hiding(&wm_high) ||
2244                     (rdev->disp_priority == 2)) {
2245                         DRM_DEBUG_KMS("force priority a to high\n");
2246                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2247                 }
2248                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2249                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2250                     !evergreen_check_latency_hiding(&wm_low) ||
2251                     (rdev->disp_priority == 2)) {
2252                         DRM_DEBUG_KMS("force priority b to high\n");
2253                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2254                 }
2255
2256                 a.full = dfixed_const(1000);
2257                 b.full = dfixed_const(mode->clock);
2258                 b.full = dfixed_div(b, a);
2259                 c.full = dfixed_const(latency_watermark_a);
2260                 c.full = dfixed_mul(c, b);
2261                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2262                 c.full = dfixed_div(c, a);
2263                 a.full = dfixed_const(16);
2264                 c.full = dfixed_div(c, a);
2265                 priority_a_mark = dfixed_trunc(c);
2266                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2267
2268                 a.full = dfixed_const(1000);
2269                 b.full = dfixed_const(mode->clock);
2270                 b.full = dfixed_div(b, a);
2271                 c.full = dfixed_const(latency_watermark_b);
2272                 c.full = dfixed_mul(c, b);
2273                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2274                 c.full = dfixed_div(c, a);
2275                 a.full = dfixed_const(16);
2276                 c.full = dfixed_div(c, a);
2277                 priority_b_mark = dfixed_trunc(c);
2278                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2279
2280                 /* Save number of lines the linebuffer leads before the scanout */
2281                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2282         }
2283
2284         /* select wm A */
2285         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2286         tmp = arb_control3;
2287         tmp &= ~LATENCY_WATERMARK_MASK(3);
2288         tmp |= LATENCY_WATERMARK_MASK(1);
2289         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2290         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2291                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2292                 LATENCY_HIGH_WATERMARK(line_time)));
2293         /* select wm B */
2294         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2295         tmp &= ~LATENCY_WATERMARK_MASK(3);
2296         tmp |= LATENCY_WATERMARK_MASK(2);
2297         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2298         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2299                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2300                 LATENCY_HIGH_WATERMARK(line_time)));
2301         /* restore original selection */
2302         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2303
2304         /* write the priority marks */
2305         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2306         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2307
2308         /* save values for DPM */
2309         radeon_crtc->line_time = line_time;
2310         radeon_crtc->wm_high = latency_watermark_a;
2311         radeon_crtc->wm_low = latency_watermark_b;
2312 }
2313
2314 /**
2315  * evergreen_bandwidth_update - update display watermarks callback.
2316  *
2317  * @rdev: radeon_device pointer
2318  *
2319  * Update the display watermarks based on the requested mode(s)
2320  * (evergreen+).
2321  */
2322 void evergreen_bandwidth_update(struct radeon_device *rdev)
2323 {
2324         struct drm_display_mode *mode0 = NULL;
2325         struct drm_display_mode *mode1 = NULL;
2326         u32 num_heads = 0, lb_size;
2327         int i;
2328
2329         if (!rdev->mode_info.mode_config_initialized)
2330                 return;
2331
2332         radeon_update_display_priority(rdev);
2333
2334         for (i = 0; i < rdev->num_crtc; i++) {
2335                 if (rdev->mode_info.crtcs[i]->base.enabled)
2336                         num_heads++;
2337         }
2338         for (i = 0; i < rdev->num_crtc; i += 2) {
2339                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2340                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2341                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2342                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2343                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2344                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2345         }
2346 }
2347
2348 /**
2349  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2350  *
2351  * @rdev: radeon_device pointer
2352  *
2353  * Wait for the MC (memory controller) to be idle.
2354  * (evergreen+).
2355  * Returns 0 if the MC is idle, -1 if not.
2356  */
2357 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2358 {
2359         unsigned i;
2360         u32 tmp;
2361
2362         for (i = 0; i < rdev->usec_timeout; i++) {
2363                 /* read MC_STATUS */
2364                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2365                 if (!tmp)
2366                         return 0;
2367                 udelay(1);
2368         }
2369         return -1;
2370 }
2371
2372 /*
2373  * GART
2374  */
2375 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2376 {
2377         unsigned i;
2378         u32 tmp;
2379
2380         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2381
2382         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2383         for (i = 0; i < rdev->usec_timeout; i++) {
2384                 /* read MC_STATUS */
2385                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2386                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2387                 if (tmp == 2) {
2388                         pr_warn("[drm] r600 flush TLB failed\n");
2389                         return;
2390                 }
2391                 if (tmp) {
2392                         return;
2393                 }
2394                 udelay(1);
2395         }
2396 }
2397
2398 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2399 {
2400         u32 tmp;
2401         int r;
2402
2403         if (rdev->gart.robj == NULL) {
2404                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2405                 return -EINVAL;
2406         }
2407         r = radeon_gart_table_vram_pin(rdev);
2408         if (r)
2409                 return r;
2410         /* Setup L2 cache */
2411         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2412                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2413                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2414         WREG32(VM_L2_CNTL2, 0);
2415         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2416         /* Setup TLB control */
2417         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2418                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2419                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2420                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2421         if (rdev->flags & RADEON_IS_IGP) {
2422                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2423                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2424                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2425         } else {
2426                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2427                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2428                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2429                 if ((rdev->family == CHIP_JUNIPER) ||
2430                     (rdev->family == CHIP_CYPRESS) ||
2431                     (rdev->family == CHIP_HEMLOCK) ||
2432                     (rdev->family == CHIP_BARTS))
2433                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2434         }
2435         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2436         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2437         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2438         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2439         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2440         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2441         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2442         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2443                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2444         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2445                         (u32)(rdev->dummy_page.addr >> 12));
2446         WREG32(VM_CONTEXT1_CNTL, 0);
2447
2448         evergreen_pcie_gart_tlb_flush(rdev);
2449         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2450                  (unsigned)(rdev->mc.gtt_size >> 20),
2451                  (unsigned long long)rdev->gart.table_addr);
2452         rdev->gart.ready = true;
2453         return 0;
2454 }
2455
2456 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2457 {
2458         u32 tmp;
2459
2460         /* Disable all tables */
2461         WREG32(VM_CONTEXT0_CNTL, 0);
2462         WREG32(VM_CONTEXT1_CNTL, 0);
2463
2464         /* Setup L2 cache */
2465         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2466                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2467         WREG32(VM_L2_CNTL2, 0);
2468         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2469         /* Setup TLB control */
2470         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2471         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2472         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2473         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2474         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2475         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2476         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2477         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2478         radeon_gart_table_vram_unpin(rdev);
2479 }
2480
2481 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2482 {
2483         evergreen_pcie_gart_disable(rdev);
2484         radeon_gart_table_vram_free(rdev);
2485         radeon_gart_fini(rdev);
2486 }
2487
2488
2489 static void evergreen_agp_enable(struct radeon_device *rdev)
2490 {
2491         u32 tmp;
2492
2493         /* Setup L2 cache */
2494         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2495                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2496                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2497         WREG32(VM_L2_CNTL2, 0);
2498         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2499         /* Setup TLB control */
2500         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2501                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2502                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2503                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2504         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2505         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2506         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2507         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2508         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2509         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2510         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2511         WREG32(VM_CONTEXT0_CNTL, 0);
2512         WREG32(VM_CONTEXT1_CNTL, 0);
2513 }
2514
2515 static const unsigned ni_dig_offsets[] =
2516 {
2517         NI_DIG0_REGISTER_OFFSET,
2518         NI_DIG1_REGISTER_OFFSET,
2519         NI_DIG2_REGISTER_OFFSET,
2520         NI_DIG3_REGISTER_OFFSET,
2521         NI_DIG4_REGISTER_OFFSET,
2522         NI_DIG5_REGISTER_OFFSET
2523 };
2524
2525 static const unsigned ni_tx_offsets[] =
2526 {
2527         NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2528         NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2529         NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2530         NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2531         NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2532         NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2533 };
2534
2535 static const unsigned evergreen_dp_offsets[] =
2536 {
2537         EVERGREEN_DP0_REGISTER_OFFSET,
2538         EVERGREEN_DP1_REGISTER_OFFSET,
2539         EVERGREEN_DP2_REGISTER_OFFSET,
2540         EVERGREEN_DP3_REGISTER_OFFSET,
2541         EVERGREEN_DP4_REGISTER_OFFSET,
2542         EVERGREEN_DP5_REGISTER_OFFSET
2543 };
2544
2545 static const unsigned evergreen_disp_int_status[] =
2546 {
2547         DISP_INTERRUPT_STATUS,
2548         DISP_INTERRUPT_STATUS_CONTINUE,
2549         DISP_INTERRUPT_STATUS_CONTINUE2,
2550         DISP_INTERRUPT_STATUS_CONTINUE3,
2551         DISP_INTERRUPT_STATUS_CONTINUE4,
2552         DISP_INTERRUPT_STATUS_CONTINUE5
2553 };
2554
2555 /*
2556  * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2557  * We go from crtc to connector and it is not relible  since it
2558  * should be an opposite direction .If crtc is enable then
2559  * find the dig_fe which selects this crtc and insure that it enable.
2560  * if such dig_fe is found then find dig_be which selects found dig_be and
2561  * insure that it enable and in DP_SST mode.
2562  * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2563  * from dp symbols clocks .
2564  */
2565 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2566                                                unsigned crtc_id, unsigned *ret_dig_fe)
2567 {
2568         unsigned i;
2569         unsigned dig_fe;
2570         unsigned dig_be;
2571         unsigned dig_en_be;
2572         unsigned uniphy_pll;
2573         unsigned digs_fe_selected;
2574         unsigned dig_be_mode;
2575         unsigned dig_fe_mask;
2576         bool is_enabled = false;
2577         bool found_crtc = false;
2578
2579         /* loop through all running dig_fe to find selected crtc */
2580         for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2581                 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2582                 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2583                     crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2584                         /* found running pipe */
2585                         found_crtc = true;
2586                         dig_fe_mask = 1 << i;
2587                         dig_fe = i;
2588                         break;
2589                 }
2590         }
2591
2592         if (found_crtc) {
2593                 /* loop through all running dig_be to find selected dig_fe */
2594                 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2595                         dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2596                         /* if dig_fe_selected by dig_be? */
2597                         digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2598                         dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2599                         if (dig_fe_mask &  digs_fe_selected &&
2600                             /* if dig_be in sst mode? */
2601                             dig_be_mode == NI_DIG_BE_DPSST) {
2602                                 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2603                                                    ni_dig_offsets[i]);
2604                                 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2605                                                     ni_tx_offsets[i]);
2606                                 /* dig_be enable and tx is running */
2607                                 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2608                                     dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2609                                     uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2610                                         is_enabled = true;
2611                                         *ret_dig_fe = dig_fe;
2612                                         break;
2613                                 }
2614                         }
2615                 }
2616         }
2617
2618         return is_enabled;
2619 }
2620
2621 /*
2622  * Blank dig when in dp sst mode
2623  * Dig ignores crtc timing
2624  */
2625 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2626                                       unsigned dig_fe)
2627 {
2628         unsigned stream_ctrl;
2629         unsigned fifo_ctrl;
2630         unsigned counter = 0;
2631
2632         if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2633                 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2634                 return;
2635         }
2636
2637         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2638                              evergreen_dp_offsets[dig_fe]);
2639         if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2640                 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2641                 return;
2642         }
2643
2644         stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2645         WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2646                evergreen_dp_offsets[dig_fe], stream_ctrl);
2647
2648         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2649                              evergreen_dp_offsets[dig_fe]);
2650         while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2651                 msleep(1);
2652                 counter++;
2653                 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2654                                      evergreen_dp_offsets[dig_fe]);
2655         }
2656         if (counter >= 32 )
2657                 DRM_ERROR("counter exceeds %d\n", counter);
2658
2659         fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2660         fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2661         WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2662
2663 }
2664
2665 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2666 {
2667         u32 crtc_enabled, tmp, frame_count, blackout;
2668         int i, j;
2669         unsigned dig_fe;
2670
2671         if (!ASIC_IS_NODCE(rdev)) {
2672                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2673                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2674
2675                 /* disable VGA render */
2676                 WREG32(VGA_RENDER_CONTROL, 0);
2677         }
2678         /* blank the display controllers */
2679         for (i = 0; i < rdev->num_crtc; i++) {
2680                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2681                 if (crtc_enabled) {
2682                         save->crtc_enabled[i] = true;
2683                         if (ASIC_IS_DCE6(rdev)) {
2684                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2685                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2686                                         radeon_wait_for_vblank(rdev, i);
2687                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2688                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2689                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2690                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2691                                 }
2692                         } else {
2693                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2694                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2695                                         radeon_wait_for_vblank(rdev, i);
2696                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2697                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2698                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2699                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2700                                 }
2701                         }
2702                         /* wait for the next frame */
2703                         frame_count = radeon_get_vblank_counter(rdev, i);
2704                         for (j = 0; j < rdev->usec_timeout; j++) {
2705                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2706                                         break;
2707                                 udelay(1);
2708                         }
2709                         /*we should disable dig if it drives dp sst*/
2710                         /*but we are in radeon_device_init and the topology is unknown*/
2711                         /*and it is available after radeon_modeset_init*/
2712                         /*the following method radeon_atom_encoder_dpms_dig*/
2713                         /*does the job if we initialize it properly*/
2714                         /*for now we do it this manually*/
2715                         /**/
2716                         if (ASIC_IS_DCE5(rdev) &&
2717                             evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2718                                 evergreen_blank_dp_output(rdev, dig_fe);
2719                         /*we could remove 6 lines below*/
2720                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2721                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2722                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2723                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2724                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2725                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2726                         save->crtc_enabled[i] = false;
2727                         /* ***** */
2728                 } else {
2729                         save->crtc_enabled[i] = false;
2730                 }
2731         }
2732
2733         radeon_mc_wait_for_idle(rdev);
2734
2735         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2736         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2737                 /* Block CPU access */
2738                 WREG32(BIF_FB_EN, 0);
2739                 /* blackout the MC */
2740                 blackout &= ~BLACKOUT_MODE_MASK;
2741                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2742         }
2743         /* wait for the MC to settle */
2744         udelay(100);
2745
2746         /* lock double buffered regs */
2747         for (i = 0; i < rdev->num_crtc; i++) {
2748                 if (save->crtc_enabled[i]) {
2749                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2750                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2751                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2752                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2753                         }
2754                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2755                         if (!(tmp & 1)) {
2756                                 tmp |= 1;
2757                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2758                         }
2759                 }
2760         }
2761 }
2762
2763 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2764 {
2765         u32 tmp, frame_count;
2766         int i, j;
2767
2768         /* update crtc base addresses */
2769         for (i = 0; i < rdev->num_crtc; i++) {
2770                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2771                        upper_32_bits(rdev->mc.vram_start));
2772                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2773                        upper_32_bits(rdev->mc.vram_start));
2774                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2775                        (u32)rdev->mc.vram_start);
2776                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2777                        (u32)rdev->mc.vram_start);
2778         }
2779
2780         if (!ASIC_IS_NODCE(rdev)) {
2781                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2782                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2783         }
2784
2785         /* unlock regs and wait for update */
2786         for (i = 0; i < rdev->num_crtc; i++) {
2787                 if (save->crtc_enabled[i]) {
2788                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2789                         if ((tmp & 0x7) != 0) {
2790                                 tmp &= ~0x7;
2791                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2792                         }
2793                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2794                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2795                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2796                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2797                         }
2798                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2799                         if (tmp & 1) {
2800                                 tmp &= ~1;
2801                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2802                         }
2803                         for (j = 0; j < rdev->usec_timeout; j++) {
2804                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2805                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2806                                         break;
2807                                 udelay(1);
2808                         }
2809                 }
2810         }
2811
2812         /* unblackout the MC */
2813         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2814         tmp &= ~BLACKOUT_MODE_MASK;
2815         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2816         /* allow CPU access */
2817         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2818
2819         for (i = 0; i < rdev->num_crtc; i++) {
2820                 if (save->crtc_enabled[i]) {
2821                         if (ASIC_IS_DCE6(rdev)) {
2822                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2823                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2824                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2825                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2826                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2827                         } else {
2828                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2829                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2830                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2831                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2832                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2833                         }
2834                         /* wait for the next frame */
2835                         frame_count = radeon_get_vblank_counter(rdev, i);
2836                         for (j = 0; j < rdev->usec_timeout; j++) {
2837                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2838                                         break;
2839                                 udelay(1);
2840                         }
2841                 }
2842         }
2843         if (!ASIC_IS_NODCE(rdev)) {
2844                 /* Unlock vga access */
2845                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2846                 mdelay(1);
2847                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2848         }
2849 }
2850
2851 void evergreen_mc_program(struct radeon_device *rdev)
2852 {
2853         struct evergreen_mc_save save;
2854         u32 tmp;
2855         int i, j;
2856
2857         /* Initialize HDP */
2858         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2859                 WREG32((0x2c14 + j), 0x00000000);
2860                 WREG32((0x2c18 + j), 0x00000000);
2861                 WREG32((0x2c1c + j), 0x00000000);
2862                 WREG32((0x2c20 + j), 0x00000000);
2863                 WREG32((0x2c24 + j), 0x00000000);
2864         }
2865         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2866
2867         evergreen_mc_stop(rdev, &save);
2868         if (evergreen_mc_wait_for_idle(rdev)) {
2869                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2870         }
2871         /* Lockout access through VGA aperture*/
2872         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2873         /* Update configuration */
2874         if (rdev->flags & RADEON_IS_AGP) {
2875                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2876                         /* VRAM before AGP */
2877                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2878                                 rdev->mc.vram_start >> 12);
2879                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2880                                 rdev->mc.gtt_end >> 12);
2881                 } else {
2882                         /* VRAM after AGP */
2883                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2884                                 rdev->mc.gtt_start >> 12);
2885                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2886                                 rdev->mc.vram_end >> 12);
2887                 }
2888         } else {
2889                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2890                         rdev->mc.vram_start >> 12);
2891                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2892                         rdev->mc.vram_end >> 12);
2893         }
2894         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2895         /* llano/ontario only */
2896         if ((rdev->family == CHIP_PALM) ||
2897             (rdev->family == CHIP_SUMO) ||
2898             (rdev->family == CHIP_SUMO2)) {
2899                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2900                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2901                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2902                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2903         }
2904         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2905         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2906         WREG32(MC_VM_FB_LOCATION, tmp);
2907         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2908         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2909         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2910         if (rdev->flags & RADEON_IS_AGP) {
2911                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2912                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2913                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2914         } else {
2915                 WREG32(MC_VM_AGP_BASE, 0);
2916                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2917                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2918         }
2919         if (evergreen_mc_wait_for_idle(rdev)) {
2920                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2921         }
2922         evergreen_mc_resume(rdev, &save);
2923         /* we need to own VRAM, so turn off the VGA renderer here
2924          * to stop it overwriting our objects */
2925         rv515_vga_render_disable(rdev);
2926 }
2927
2928 /*
2929  * CP.
2930  */
2931 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2932 {
2933         struct radeon_ring *ring = &rdev->ring[ib->ring];
2934         u32 next_rptr;
2935
2936         /* set to DX10/11 mode */
2937         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2938         radeon_ring_write(ring, 1);
2939
2940         if (ring->rptr_save_reg) {
2941                 next_rptr = ring->wptr + 3 + 4;
2942                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2943                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2944                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2945                 radeon_ring_write(ring, next_rptr);
2946         } else if (rdev->wb.enabled) {
2947                 next_rptr = ring->wptr + 5 + 4;
2948                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2949                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2950                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2951                 radeon_ring_write(ring, next_rptr);
2952                 radeon_ring_write(ring, 0);
2953         }
2954
2955         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2956         radeon_ring_write(ring,
2957 #ifdef __BIG_ENDIAN
2958                           (2 << 0) |
2959 #endif
2960                           (ib->gpu_addr & 0xFFFFFFFC));
2961         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2962         radeon_ring_write(ring, ib->length_dw);
2963 }
2964
2965
2966 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2967 {
2968         const __be32 *fw_data;
2969         int i;
2970
2971         if (!rdev->me_fw || !rdev->pfp_fw)
2972                 return -EINVAL;
2973
2974         r700_cp_stop(rdev);
2975         WREG32(CP_RB_CNTL,
2976 #ifdef __BIG_ENDIAN
2977                BUF_SWAP_32BIT |
2978 #endif
2979                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2980
2981         fw_data = (const __be32 *)rdev->pfp_fw->data;
2982         WREG32(CP_PFP_UCODE_ADDR, 0);
2983         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2984                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2985         WREG32(CP_PFP_UCODE_ADDR, 0);
2986
2987         fw_data = (const __be32 *)rdev->me_fw->data;
2988         WREG32(CP_ME_RAM_WADDR, 0);
2989         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2990                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2991
2992         WREG32(CP_PFP_UCODE_ADDR, 0);
2993         WREG32(CP_ME_RAM_WADDR, 0);
2994         WREG32(CP_ME_RAM_RADDR, 0);
2995         return 0;
2996 }
2997
2998 static int evergreen_cp_start(struct radeon_device *rdev)
2999 {
3000         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3001         int r, i;
3002         uint32_t cp_me;
3003
3004         r = radeon_ring_lock(rdev, ring, 7);
3005         if (r) {
3006                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3007                 return r;
3008         }
3009         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3010         radeon_ring_write(ring, 0x1);
3011         radeon_ring_write(ring, 0x0);
3012         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3013         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3014         radeon_ring_write(ring, 0);
3015         radeon_ring_write(ring, 0);
3016         radeon_ring_unlock_commit(rdev, ring, false);
3017
3018         cp_me = 0xff;
3019         WREG32(CP_ME_CNTL, cp_me);
3020
3021         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3022         if (r) {
3023                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3024                 return r;
3025         }
3026
3027         /* setup clear context state */
3028         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3029         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3030
3031         for (i = 0; i < evergreen_default_size; i++)
3032                 radeon_ring_write(ring, evergreen_default_state[i]);
3033
3034         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3035         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3036
3037         /* set clear context state */
3038         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3039         radeon_ring_write(ring, 0);
3040
3041         /* SQ_VTX_BASE_VTX_LOC */
3042         radeon_ring_write(ring, 0xc0026f00);
3043         radeon_ring_write(ring, 0x00000000);
3044         radeon_ring_write(ring, 0x00000000);
3045         radeon_ring_write(ring, 0x00000000);
3046
3047         /* Clear consts */
3048         radeon_ring_write(ring, 0xc0036f00);
3049         radeon_ring_write(ring, 0x00000bc4);
3050         radeon_ring_write(ring, 0xffffffff);
3051         radeon_ring_write(ring, 0xffffffff);
3052         radeon_ring_write(ring, 0xffffffff);
3053
3054         radeon_ring_write(ring, 0xc0026900);
3055         radeon_ring_write(ring, 0x00000316);
3056         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3057         radeon_ring_write(ring, 0x00000010); /*  */
3058
3059         radeon_ring_unlock_commit(rdev, ring, false);
3060
3061         return 0;
3062 }
3063
3064 static int evergreen_cp_resume(struct radeon_device *rdev)
3065 {
3066         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3067         u32 tmp;
3068         u32 rb_bufsz;
3069         int r;
3070
3071         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3072         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3073                                  SOFT_RESET_PA |
3074                                  SOFT_RESET_SH |
3075                                  SOFT_RESET_VGT |
3076                                  SOFT_RESET_SPI |
3077                                  SOFT_RESET_SX));
3078         RREG32(GRBM_SOFT_RESET);
3079         mdelay(15);
3080         WREG32(GRBM_SOFT_RESET, 0);
3081         RREG32(GRBM_SOFT_RESET);
3082
3083         /* Set ring buffer size */
3084         rb_bufsz = order_base_2(ring->ring_size / 8);
3085         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3086 #ifdef __BIG_ENDIAN
3087         tmp |= BUF_SWAP_32BIT;
3088 #endif
3089         WREG32(CP_RB_CNTL, tmp);
3090         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3091         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3092
3093         /* Set the write pointer delay */
3094         WREG32(CP_RB_WPTR_DELAY, 0);
3095
3096         /* Initialize the ring buffer's read and write pointers */
3097         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3098         WREG32(CP_RB_RPTR_WR, 0);
3099         ring->wptr = 0;
3100         WREG32(CP_RB_WPTR, ring->wptr);
3101
3102         /* set the wb address whether it's enabled or not */
3103         WREG32(CP_RB_RPTR_ADDR,
3104                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3105         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3106         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3107
3108         if (rdev->wb.enabled)
3109                 WREG32(SCRATCH_UMSK, 0xff);
3110         else {
3111                 tmp |= RB_NO_UPDATE;
3112                 WREG32(SCRATCH_UMSK, 0);
3113         }
3114
3115         mdelay(1);
3116         WREG32(CP_RB_CNTL, tmp);
3117
3118         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3119         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3120
3121         evergreen_cp_start(rdev);
3122         ring->ready = true;
3123         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3124         if (r) {
3125                 ring->ready = false;
3126                 return r;
3127         }
3128         return 0;
3129 }
3130
3131 /*
3132  * Core functions
3133  */
3134 static void evergreen_gpu_init(struct radeon_device *rdev)
3135 {
3136         u32 gb_addr_config;
3137         u32 mc_arb_ramcfg;
3138         u32 sx_debug_1;
3139         u32 smx_dc_ctl0;
3140         u32 sq_config;
3141         u32 sq_lds_resource_mgmt;
3142         u32 sq_gpr_resource_mgmt_1;
3143         u32 sq_gpr_resource_mgmt_2;
3144         u32 sq_gpr_resource_mgmt_3;
3145         u32 sq_thread_resource_mgmt;
3146         u32 sq_thread_resource_mgmt_2;
3147         u32 sq_stack_resource_mgmt_1;
3148         u32 sq_stack_resource_mgmt_2;
3149         u32 sq_stack_resource_mgmt_3;
3150         u32 vgt_cache_invalidation;
3151         u32 hdp_host_path_cntl, tmp;
3152         u32 disabled_rb_mask;
3153         int i, j, ps_thread_count;
3154
3155         switch (rdev->family) {
3156         case CHIP_CYPRESS:
3157         case CHIP_HEMLOCK:
3158                 rdev->config.evergreen.num_ses = 2;
3159                 rdev->config.evergreen.max_pipes = 4;
3160                 rdev->config.evergreen.max_tile_pipes = 8;
3161                 rdev->config.evergreen.max_simds = 10;
3162                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3163                 rdev->config.evergreen.max_gprs = 256;
3164                 rdev->config.evergreen.max_threads = 248;
3165                 rdev->config.evergreen.max_gs_threads = 32;
3166                 rdev->config.evergreen.max_stack_entries = 512;
3167                 rdev->config.evergreen.sx_num_of_sets = 4;
3168                 rdev->config.evergreen.sx_max_export_size = 256;
3169                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3170                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3171                 rdev->config.evergreen.max_hw_contexts = 8;
3172                 rdev->config.evergreen.sq_num_cf_insts = 2;
3173
3174                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3175                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3177                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3178                 break;
3179         case CHIP_JUNIPER:
3180                 rdev->config.evergreen.num_ses = 1;
3181                 rdev->config.evergreen.max_pipes = 4;
3182                 rdev->config.evergreen.max_tile_pipes = 4;
3183                 rdev->config.evergreen.max_simds = 10;
3184                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3185                 rdev->config.evergreen.max_gprs = 256;
3186                 rdev->config.evergreen.max_threads = 248;
3187                 rdev->config.evergreen.max_gs_threads = 32;
3188                 rdev->config.evergreen.max_stack_entries = 512;
3189                 rdev->config.evergreen.sx_num_of_sets = 4;
3190                 rdev->config.evergreen.sx_max_export_size = 256;
3191                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3192                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3193                 rdev->config.evergreen.max_hw_contexts = 8;
3194                 rdev->config.evergreen.sq_num_cf_insts = 2;
3195
3196                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3197                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3198                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3199                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3200                 break;
3201         case CHIP_REDWOOD:
3202                 rdev->config.evergreen.num_ses = 1;
3203                 rdev->config.evergreen.max_pipes = 4;
3204                 rdev->config.evergreen.max_tile_pipes = 4;
3205                 rdev->config.evergreen.max_simds = 5;
3206                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3207                 rdev->config.evergreen.max_gprs = 256;
3208                 rdev->config.evergreen.max_threads = 248;
3209                 rdev->config.evergreen.max_gs_threads = 32;
3210                 rdev->config.evergreen.max_stack_entries = 256;
3211                 rdev->config.evergreen.sx_num_of_sets = 4;
3212                 rdev->config.evergreen.sx_max_export_size = 256;
3213                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3214                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3215                 rdev->config.evergreen.max_hw_contexts = 8;
3216                 rdev->config.evergreen.sq_num_cf_insts = 2;
3217
3218                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3219                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3220                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3221                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3222                 break;
3223         case CHIP_CEDAR:
3224         default:
3225                 rdev->config.evergreen.num_ses = 1;
3226                 rdev->config.evergreen.max_pipes = 2;
3227                 rdev->config.evergreen.max_tile_pipes = 2;
3228                 rdev->config.evergreen.max_simds = 2;
3229                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3230                 rdev->config.evergreen.max_gprs = 256;
3231                 rdev->config.evergreen.max_threads = 192;
3232                 rdev->config.evergreen.max_gs_threads = 16;
3233                 rdev->config.evergreen.max_stack_entries = 256;
3234                 rdev->config.evergreen.sx_num_of_sets = 4;
3235                 rdev->config.evergreen.sx_max_export_size = 128;
3236                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3237                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3238                 rdev->config.evergreen.max_hw_contexts = 4;
3239                 rdev->config.evergreen.sq_num_cf_insts = 1;
3240
3241                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3242                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3243                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3244                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3245                 break;
3246         case CHIP_PALM:
3247                 rdev->config.evergreen.num_ses = 1;
3248                 rdev->config.evergreen.max_pipes = 2;
3249                 rdev->config.evergreen.max_tile_pipes = 2;
3250                 rdev->config.evergreen.max_simds = 2;
3251                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3252                 rdev->config.evergreen.max_gprs = 256;
3253                 rdev->config.evergreen.max_threads = 192;
3254                 rdev->config.evergreen.max_gs_threads = 16;
3255                 rdev->config.evergreen.max_stack_entries = 256;
3256                 rdev->config.evergreen.sx_num_of_sets = 4;
3257                 rdev->config.evergreen.sx_max_export_size = 128;
3258                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3259                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3260                 rdev->config.evergreen.max_hw_contexts = 4;
3261                 rdev->config.evergreen.sq_num_cf_insts = 1;
3262
3263                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3264                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3265                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3266                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3267                 break;
3268         case CHIP_SUMO:
3269                 rdev->config.evergreen.num_ses = 1;
3270                 rdev->config.evergreen.max_pipes = 4;
3271                 rdev->config.evergreen.max_tile_pipes = 4;
3272                 if (rdev->pdev->device == 0x9648)
3273                         rdev->config.evergreen.max_simds = 3;
3274                 else if ((rdev->pdev->device == 0x9647) ||
3275                          (rdev->pdev->device == 0x964a))
3276                         rdev->config.evergreen.max_simds = 4;
3277                 else
3278                         rdev->config.evergreen.max_simds = 5;
3279                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3280                 rdev->config.evergreen.max_gprs = 256;
3281                 rdev->config.evergreen.max_threads = 248;
3282                 rdev->config.evergreen.max_gs_threads = 32;
3283                 rdev->config.evergreen.max_stack_entries = 256;
3284                 rdev->config.evergreen.sx_num_of_sets = 4;
3285                 rdev->config.evergreen.sx_max_export_size = 256;
3286                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3287                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3288                 rdev->config.evergreen.max_hw_contexts = 8;
3289                 rdev->config.evergreen.sq_num_cf_insts = 2;
3290
3291                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3292                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3293                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3294                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3295                 break;
3296         case CHIP_SUMO2:
3297                 rdev->config.evergreen.num_ses = 1;
3298                 rdev->config.evergreen.max_pipes = 4;
3299                 rdev->config.evergreen.max_tile_pipes = 4;
3300                 rdev->config.evergreen.max_simds = 2;
3301                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3302                 rdev->config.evergreen.max_gprs = 256;
3303                 rdev->config.evergreen.max_threads = 248;
3304                 rdev->config.evergreen.max_gs_threads = 32;
3305                 rdev->config.evergreen.max_stack_entries = 512;
3306                 rdev->config.evergreen.sx_num_of_sets = 4;
3307                 rdev->config.evergreen.sx_max_export_size = 256;
3308                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3309                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3310                 rdev->config.evergreen.max_hw_contexts = 4;
3311                 rdev->config.evergreen.sq_num_cf_insts = 2;
3312
3313                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3314                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3315                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3316                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3317                 break;
3318         case CHIP_BARTS:
3319                 rdev->config.evergreen.num_ses = 2;
3320                 rdev->config.evergreen.max_pipes = 4;
3321                 rdev->config.evergreen.max_tile_pipes = 8;
3322                 rdev->config.evergreen.max_simds = 7;
3323                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3324                 rdev->config.evergreen.max_gprs = 256;
3325                 rdev->config.evergreen.max_threads = 248;
3326                 rdev->config.evergreen.max_gs_threads = 32;
3327                 rdev->config.evergreen.max_stack_entries = 512;
3328                 rdev->config.evergreen.sx_num_of_sets = 4;
3329                 rdev->config.evergreen.sx_max_export_size = 256;
3330                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3331                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3332                 rdev->config.evergreen.max_hw_contexts = 8;
3333                 rdev->config.evergreen.sq_num_cf_insts = 2;
3334
3335                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3336                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3337                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3338                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3339                 break;
3340         case CHIP_TURKS:
3341                 rdev->config.evergreen.num_ses = 1;
3342                 rdev->config.evergreen.max_pipes = 4;
3343                 rdev->config.evergreen.max_tile_pipes = 4;
3344                 rdev->config.evergreen.max_simds = 6;
3345                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3346                 rdev->config.evergreen.max_gprs = 256;
3347                 rdev->config.evergreen.max_threads = 248;
3348                 rdev->config.evergreen.max_gs_threads = 32;
3349                 rdev->config.evergreen.max_stack_entries = 256;
3350                 rdev->config.evergreen.sx_num_of_sets = 4;
3351                 rdev->config.evergreen.sx_max_export_size = 256;
3352                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3353                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3354                 rdev->config.evergreen.max_hw_contexts = 8;
3355                 rdev->config.evergreen.sq_num_cf_insts = 2;
3356
3357                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3358                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3359                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3360                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3361                 break;
3362         case CHIP_CAICOS:
3363                 rdev->config.evergreen.num_ses = 1;
3364                 rdev->config.evergreen.max_pipes = 2;
3365                 rdev->config.evergreen.max_tile_pipes = 2;
3366                 rdev->config.evergreen.max_simds = 2;
3367                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3368                 rdev->config.evergreen.max_gprs = 256;
3369                 rdev->config.evergreen.max_threads = 192;
3370                 rdev->config.evergreen.max_gs_threads = 16;
3371                 rdev->config.evergreen.max_stack_entries = 256;
3372                 rdev->config.evergreen.sx_num_of_sets = 4;
3373                 rdev->config.evergreen.sx_max_export_size = 128;
3374                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3375                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3376                 rdev->config.evergreen.max_hw_contexts = 4;
3377                 rdev->config.evergreen.sq_num_cf_insts = 1;
3378
3379                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3380                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3381                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3382                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3383                 break;
3384         }
3385
3386         /* Initialize HDP */
3387         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3388                 WREG32((0x2c14 + j), 0x00000000);
3389                 WREG32((0x2c18 + j), 0x00000000);
3390                 WREG32((0x2c1c + j), 0x00000000);
3391                 WREG32((0x2c20 + j), 0x00000000);
3392                 WREG32((0x2c24 + j), 0x00000000);
3393         }
3394
3395         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3396         WREG32(SRBM_INT_CNTL, 0x1);
3397         WREG32(SRBM_INT_ACK, 0x1);
3398
3399         evergreen_fix_pci_max_read_req_size(rdev);
3400
3401         RREG32(MC_SHARED_CHMAP);
3402         if ((rdev->family == CHIP_PALM) ||
3403             (rdev->family == CHIP_SUMO) ||
3404             (rdev->family == CHIP_SUMO2))
3405                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3406         else
3407                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3408
3409         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3410          * not have bank info, so create a custom tiling dword.
3411          * bits 3:0   num_pipes
3412          * bits 7:4   num_banks
3413          * bits 11:8  group_size
3414          * bits 15:12 row_size
3415          */
3416         rdev->config.evergreen.tile_config = 0;
3417         switch (rdev->config.evergreen.max_tile_pipes) {
3418         case 1:
3419         default:
3420                 rdev->config.evergreen.tile_config |= (0 << 0);
3421                 break;
3422         case 2:
3423                 rdev->config.evergreen.tile_config |= (1 << 0);
3424                 break;
3425         case 4:
3426                 rdev->config.evergreen.tile_config |= (2 << 0);
3427                 break;
3428         case 8:
3429                 rdev->config.evergreen.tile_config |= (3 << 0);
3430                 break;
3431         }
3432         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3433         if (rdev->flags & RADEON_IS_IGP)
3434                 rdev->config.evergreen.tile_config |= 1 << 4;
3435         else {
3436                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3437                 case 0: /* four banks */
3438                         rdev->config.evergreen.tile_config |= 0 << 4;
3439                         break;
3440                 case 1: /* eight banks */
3441                         rdev->config.evergreen.tile_config |= 1 << 4;
3442                         break;
3443                 case 2: /* sixteen banks */
3444                 default:
3445                         rdev->config.evergreen.tile_config |= 2 << 4;
3446                         break;
3447                 }
3448         }
3449         rdev->config.evergreen.tile_config |= 0 << 8;
3450         rdev->config.evergreen.tile_config |=
3451                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3452
3453         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3454                 u32 efuse_straps_4;
3455                 u32 efuse_straps_3;
3456
3457                 efuse_straps_4 = RREG32_RCU(0x204);
3458                 efuse_straps_3 = RREG32_RCU(0x203);
3459                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3460                       ((efuse_straps_3 & 0xf0000000) >> 28));
3461         } else {
3462                 tmp = 0;
3463                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3464                         u32 rb_disable_bitmap;
3465
3466                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3467                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3468                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3469                         tmp <<= 4;
3470                         tmp |= rb_disable_bitmap;
3471                 }
3472         }
3473         /* enabled rb are just the one not disabled :) */
3474         disabled_rb_mask = tmp;
3475         tmp = 0;
3476         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3477                 tmp |= (1 << i);
3478         /* if all the backends are disabled, fix it up here */
3479         if ((disabled_rb_mask & tmp) == tmp) {
3480                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3481                         disabled_rb_mask &= ~(1 << i);
3482         }
3483
3484         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3485                 u32 simd_disable_bitmap;
3486
3487                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3488                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3489                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3490                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3491                 tmp <<= 16;
3492                 tmp |= simd_disable_bitmap;
3493         }
3494         rdev->config.evergreen.active_simds = hweight32(~tmp);
3495
3496         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3497         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3498
3499         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3500         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3501         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3502         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3503         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3504         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3505         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3506
3507         if ((rdev->config.evergreen.max_backends == 1) &&
3508             (rdev->flags & RADEON_IS_IGP)) {
3509                 if ((disabled_rb_mask & 3) == 1) {
3510                         /* RB0 disabled, RB1 enabled */
3511                         tmp = 0x11111111;
3512                 } else {
3513                         /* RB1 disabled, RB0 enabled */
3514                         tmp = 0x00000000;
3515                 }
3516         } else {
3517                 tmp = gb_addr_config & NUM_PIPES_MASK;
3518                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3519                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3520         }
3521         rdev->config.evergreen.backend_map = tmp;
3522         WREG32(GB_BACKEND_MAP, tmp);
3523
3524         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3525         WREG32(CGTS_TCC_DISABLE, 0);
3526         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3527         WREG32(CGTS_USER_TCC_DISABLE, 0);
3528
3529         /* set HW defaults for 3D engine */
3530         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3531                                      ROQ_IB2_START(0x2b)));
3532
3533         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3534
3535         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3536                              SYNC_GRADIENT |
3537                              SYNC_WALKER |
3538                              SYNC_ALIGNER));
3539
3540         sx_debug_1 = RREG32(SX_DEBUG_1);
3541         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3542         WREG32(SX_DEBUG_1, sx_debug_1);
3543
3544
3545         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3546         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3547         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3548         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3549
3550         if (rdev->family <= CHIP_SUMO2)
3551                 WREG32(SMX_SAR_CTL0, 0x00010000);
3552
3553         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3554                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3555                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3556
3557         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3558                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3559                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3560
3561         WREG32(VGT_NUM_INSTANCES, 1);
3562         WREG32(SPI_CONFIG_CNTL, 0);
3563         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3564         WREG32(CP_PERFMON_CNTL, 0);
3565
3566         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3567                                   FETCH_FIFO_HIWATER(0x4) |
3568                                   DONE_FIFO_HIWATER(0xe0) |
3569                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3570
3571         sq_config = RREG32(SQ_CONFIG);
3572         sq_config &= ~(PS_PRIO(3) |
3573                        VS_PRIO(3) |
3574                        GS_PRIO(3) |
3575                        ES_PRIO(3));
3576         sq_config |= (VC_ENABLE |
3577                       EXPORT_SRC_C |
3578                       PS_PRIO(0) |
3579                       VS_PRIO(1) |
3580                       GS_PRIO(2) |
3581                       ES_PRIO(3));
3582
3583         switch (rdev->family) {
3584         case CHIP_CEDAR:
3585         case CHIP_PALM:
3586         case CHIP_SUMO:
3587         case CHIP_SUMO2:
3588         case CHIP_CAICOS:
3589                 /* no vertex cache */
3590                 sq_config &= ~VC_ENABLE;
3591                 break;
3592         default:
3593                 break;
3594         }
3595
3596         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3597
3598         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3599         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3600         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3601         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3602         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3603         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3604         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3605
3606         switch (rdev->family) {
3607         case CHIP_CEDAR:
3608         case CHIP_PALM:
3609         case CHIP_SUMO:
3610         case CHIP_SUMO2:
3611                 ps_thread_count = 96;
3612                 break;
3613         default:
3614                 ps_thread_count = 128;
3615                 break;
3616         }
3617
3618         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3619         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3624
3625         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3631
3632         WREG32(SQ_CONFIG, sq_config);
3633         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3634         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3635         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3636         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3637         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3638         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3639         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3640         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3641         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3642         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3643
3644         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3645                                           FORCE_EOV_MAX_REZ_CNT(255)));
3646
3647         switch (rdev->family) {
3648         case CHIP_CEDAR:
3649         case CHIP_PALM:
3650         case CHIP_SUMO:
3651         case CHIP_SUMO2:
3652         case CHIP_CAICOS:
3653                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3654                 break;
3655         default:
3656                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3657                 break;
3658         }
3659         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3660         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3661
3662         WREG32(VGT_GS_VERTEX_REUSE, 16);
3663         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3664         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3665
3666         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3667         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3668
3669         WREG32(CB_PERF_CTR0_SEL_0, 0);
3670         WREG32(CB_PERF_CTR0_SEL_1, 0);
3671         WREG32(CB_PERF_CTR1_SEL_0, 0);
3672         WREG32(CB_PERF_CTR1_SEL_1, 0);
3673         WREG32(CB_PERF_CTR2_SEL_0, 0);
3674         WREG32(CB_PERF_CTR2_SEL_1, 0);
3675         WREG32(CB_PERF_CTR3_SEL_0, 0);
3676         WREG32(CB_PERF_CTR3_SEL_1, 0);
3677
3678         /* clear render buffer base addresses */
3679         WREG32(CB_COLOR0_BASE, 0);
3680         WREG32(CB_COLOR1_BASE, 0);
3681         WREG32(CB_COLOR2_BASE, 0);
3682         WREG32(CB_COLOR3_BASE, 0);
3683         WREG32(CB_COLOR4_BASE, 0);
3684         WREG32(CB_COLOR5_BASE, 0);
3685         WREG32(CB_COLOR6_BASE, 0);
3686         WREG32(CB_COLOR7_BASE, 0);
3687         WREG32(CB_COLOR8_BASE, 0);
3688         WREG32(CB_COLOR9_BASE, 0);
3689         WREG32(CB_COLOR10_BASE, 0);
3690         WREG32(CB_COLOR11_BASE, 0);
3691
3692         /* set the shader const cache sizes to 0 */
3693         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3694                 WREG32(i, 0);
3695         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3696                 WREG32(i, 0);
3697
3698         tmp = RREG32(HDP_MISC_CNTL);
3699         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3700         WREG32(HDP_MISC_CNTL, tmp);
3701
3702         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3703         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3704
3705         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3706
3707         udelay(50);
3708
3709 }
3710
3711 int evergreen_mc_init(struct radeon_device *rdev)
3712 {
3713         u32 tmp;
3714         int chansize, numchan;
3715
3716         /* Get VRAM informations */
3717         rdev->mc.vram_is_ddr = true;
3718         if ((rdev->family == CHIP_PALM) ||
3719             (rdev->family == CHIP_SUMO) ||
3720             (rdev->family == CHIP_SUMO2))
3721                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3722         else
3723                 tmp = RREG32(MC_ARB_RAMCFG);
3724         if (tmp & CHANSIZE_OVERRIDE) {
3725                 chansize = 16;
3726         } else if (tmp & CHANSIZE_MASK) {
3727                 chansize = 64;
3728         } else {
3729                 chansize = 32;
3730         }
3731         tmp = RREG32(MC_SHARED_CHMAP);
3732         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3733         case 0:
3734         default:
3735                 numchan = 1;
3736                 break;
3737         case 1:
3738                 numchan = 2;
3739                 break;
3740         case 2:
3741                 numchan = 4;
3742                 break;
3743         case 3:
3744                 numchan = 8;
3745                 break;
3746         }
3747         rdev->mc.vram_width = numchan * chansize;
3748         /* Could aper size report 0 ? */
3749         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3750         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3751         /* Setup GPU memory space */
3752         if ((rdev->family == CHIP_PALM) ||
3753             (rdev->family == CHIP_SUMO) ||
3754             (rdev->family == CHIP_SUMO2)) {
3755                 /* size in bytes on fusion */
3756                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3757                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3758         } else {
3759                 /* size in MB on evergreen/cayman/tn */
3760                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3761                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3762         }
3763         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3764         r700_vram_gtt_location(rdev, &rdev->mc);
3765         radeon_update_bandwidth_info(rdev);
3766
3767         return 0;
3768 }
3769
3770 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3771 {
3772         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3773                 RREG32(GRBM_STATUS));
3774         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3775                 RREG32(GRBM_STATUS_SE0));
3776         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3777                 RREG32(GRBM_STATUS_SE1));
3778         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3779                 RREG32(SRBM_STATUS));
3780         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3781                 RREG32(SRBM_STATUS2));
3782         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3783                 RREG32(CP_STALLED_STAT1));
3784         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3785                 RREG32(CP_STALLED_STAT2));
3786         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3787                 RREG32(CP_BUSY_STAT));
3788         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3789                 RREG32(CP_STAT));
3790         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3791                 RREG32(DMA_STATUS_REG));
3792         if (rdev->family >= CHIP_CAYMAN) {
3793                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3794                          RREG32(DMA_STATUS_REG + 0x800));
3795         }
3796 }
3797
3798 bool evergreen_is_display_hung(struct radeon_device *rdev)
3799 {
3800         u32 crtc_hung = 0;
3801         u32 crtc_status[6];
3802         u32 i, j, tmp;
3803
3804         for (i = 0; i < rdev->num_crtc; i++) {
3805                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3806                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3807                         crtc_hung |= (1 << i);
3808                 }
3809         }
3810
3811         for (j = 0; j < 10; j++) {
3812                 for (i = 0; i < rdev->num_crtc; i++) {
3813                         if (crtc_hung & (1 << i)) {
3814                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3815                                 if (tmp != crtc_status[i])
3816                                         crtc_hung &= ~(1 << i);
3817                         }
3818                 }
3819                 if (crtc_hung == 0)
3820                         return false;
3821                 udelay(100);
3822         }
3823
3824         return true;
3825 }
3826
3827 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3828 {
3829         u32 reset_mask = 0;
3830         u32 tmp;
3831
3832         /* GRBM_STATUS */
3833         tmp = RREG32(GRBM_STATUS);
3834         if (tmp & (PA_BUSY | SC_BUSY |
3835                    SH_BUSY | SX_BUSY |
3836                    TA_BUSY | VGT_BUSY |
3837                    DB_BUSY | CB_BUSY |
3838                    SPI_BUSY | VGT_BUSY_NO_DMA))
3839                 reset_mask |= RADEON_RESET_GFX;
3840
3841         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3842                    CP_BUSY | CP_COHERENCY_BUSY))
3843                 reset_mask |= RADEON_RESET_CP;
3844
3845         if (tmp & GRBM_EE_BUSY)
3846                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3847
3848         /* DMA_STATUS_REG */
3849         tmp = RREG32(DMA_STATUS_REG);
3850         if (!(tmp & DMA_IDLE))
3851                 reset_mask |= RADEON_RESET_DMA;
3852
3853         /* SRBM_STATUS2 */
3854         tmp = RREG32(SRBM_STATUS2);
3855         if (tmp & DMA_BUSY)
3856                 reset_mask |= RADEON_RESET_DMA;
3857
3858         /* SRBM_STATUS */
3859         tmp = RREG32(SRBM_STATUS);
3860         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3861                 reset_mask |= RADEON_RESET_RLC;
3862
3863         if (tmp & IH_BUSY)
3864                 reset_mask |= RADEON_RESET_IH;
3865
3866         if (tmp & SEM_BUSY)
3867                 reset_mask |= RADEON_RESET_SEM;
3868
3869         if (tmp & GRBM_RQ_PENDING)
3870                 reset_mask |= RADEON_RESET_GRBM;
3871
3872         if (tmp & VMC_BUSY)
3873                 reset_mask |= RADEON_RESET_VMC;
3874
3875         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3876                    MCC_BUSY | MCD_BUSY))
3877                 reset_mask |= RADEON_RESET_MC;
3878
3879         if (evergreen_is_display_hung(rdev))
3880                 reset_mask |= RADEON_RESET_DISPLAY;
3881
3882         /* VM_L2_STATUS */
3883         tmp = RREG32(VM_L2_STATUS);
3884         if (tmp & L2_BUSY)
3885                 reset_mask |= RADEON_RESET_VMC;
3886
3887         /* Skip MC reset as it's mostly likely not hung, just busy */
3888         if (reset_mask & RADEON_RESET_MC) {
3889                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3890                 reset_mask &= ~RADEON_RESET_MC;
3891         }
3892
3893         return reset_mask;
3894 }
3895
3896 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3897 {
3898         struct evergreen_mc_save save;
3899         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3900         u32 tmp;
3901
3902         if (reset_mask == 0)
3903                 return;
3904
3905         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3906
3907         evergreen_print_gpu_status_regs(rdev);
3908
3909         /* Disable CP parsing/prefetching */
3910         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3911
3912         if (reset_mask & RADEON_RESET_DMA) {
3913                 /* Disable DMA */
3914                 tmp = RREG32(DMA_RB_CNTL);
3915                 tmp &= ~DMA_RB_ENABLE;
3916                 WREG32(DMA_RB_CNTL, tmp);
3917         }
3918
3919         udelay(50);
3920
3921         evergreen_mc_stop(rdev, &save);
3922         if (evergreen_mc_wait_for_idle(rdev)) {
3923                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3924         }
3925
3926         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3927                 grbm_soft_reset |= SOFT_RESET_DB |
3928                         SOFT_RESET_CB |
3929                         SOFT_RESET_PA |
3930                         SOFT_RESET_SC |
3931                         SOFT_RESET_SPI |
3932                         SOFT_RESET_SX |
3933                         SOFT_RESET_SH |
3934                         SOFT_RESET_TC |
3935                         SOFT_RESET_TA |
3936                         SOFT_RESET_VC |
3937                         SOFT_RESET_VGT;
3938         }
3939
3940         if (reset_mask & RADEON_RESET_CP) {
3941                 grbm_soft_reset |= SOFT_RESET_CP |
3942                         SOFT_RESET_VGT;
3943
3944                 srbm_soft_reset |= SOFT_RESET_GRBM;
3945         }
3946
3947         if (reset_mask & RADEON_RESET_DMA)
3948                 srbm_soft_reset |= SOFT_RESET_DMA;
3949
3950         if (reset_mask & RADEON_RESET_DISPLAY)
3951                 srbm_soft_reset |= SOFT_RESET_DC;
3952
3953         if (reset_mask & RADEON_RESET_RLC)
3954                 srbm_soft_reset |= SOFT_RESET_RLC;
3955
3956         if (reset_mask & RADEON_RESET_SEM)
3957                 srbm_soft_reset |= SOFT_RESET_SEM;
3958
3959         if (reset_mask & RADEON_RESET_IH)
3960                 srbm_soft_reset |= SOFT_RESET_IH;
3961
3962         if (reset_mask & RADEON_RESET_GRBM)
3963                 srbm_soft_reset |= SOFT_RESET_GRBM;
3964
3965         if (reset_mask & RADEON_RESET_VMC)
3966                 srbm_soft_reset |= SOFT_RESET_VMC;
3967
3968         if (!(rdev->flags & RADEON_IS_IGP)) {
3969                 if (reset_mask & RADEON_RESET_MC)
3970                         srbm_soft_reset |= SOFT_RESET_MC;
3971         }
3972
3973         if (grbm_soft_reset) {
3974                 tmp = RREG32(GRBM_SOFT_RESET);
3975                 tmp |= grbm_soft_reset;
3976                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3977                 WREG32(GRBM_SOFT_RESET, tmp);
3978                 tmp = RREG32(GRBM_SOFT_RESET);
3979
3980                 udelay(50);
3981
3982                 tmp &= ~grbm_soft_reset;
3983                 WREG32(GRBM_SOFT_RESET, tmp);
3984                 tmp = RREG32(GRBM_SOFT_RESET);
3985         }
3986
3987         if (srbm_soft_reset) {
3988                 tmp = RREG32(SRBM_SOFT_RESET);
3989                 tmp |= srbm_soft_reset;
3990                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3991                 WREG32(SRBM_SOFT_RESET, tmp);
3992                 tmp = RREG32(SRBM_SOFT_RESET);
3993
3994                 udelay(50);
3995
3996                 tmp &= ~srbm_soft_reset;
3997                 WREG32(SRBM_SOFT_RESET, tmp);
3998                 tmp = RREG32(SRBM_SOFT_RESET);
3999         }
4000
4001         /* Wait a little for things to settle down */
4002         udelay(50);
4003
4004         evergreen_mc_resume(rdev, &save);
4005         udelay(50);
4006
4007         evergreen_print_gpu_status_regs(rdev);
4008 }
4009
4010 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4011 {
4012         struct evergreen_mc_save save;
4013         u32 tmp, i;
4014
4015         dev_info(rdev->dev, "GPU pci config reset\n");
4016
4017         /* disable dpm? */
4018
4019         /* Disable CP parsing/prefetching */
4020         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4021         udelay(50);
4022         /* Disable DMA */
4023         tmp = RREG32(DMA_RB_CNTL);
4024         tmp &= ~DMA_RB_ENABLE;
4025         WREG32(DMA_RB_CNTL, tmp);
4026         /* XXX other engines? */
4027
4028         /* halt the rlc */
4029         r600_rlc_stop(rdev);
4030
4031         udelay(50);
4032
4033         /* set mclk/sclk to bypass */
4034         rv770_set_clk_bypass_mode(rdev);
4035         /* disable BM */
4036         pci_clear_master(rdev->pdev);
4037         /* disable mem access */
4038         evergreen_mc_stop(rdev, &save);
4039         if (evergreen_mc_wait_for_idle(rdev)) {
4040                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4041         }
4042         /* reset */
4043         radeon_pci_config_reset(rdev);
4044         /* wait for asic to come out of reset */
4045         for (i = 0; i < rdev->usec_timeout; i++) {
4046                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4047                         break;
4048                 udelay(1);
4049         }
4050 }
4051
4052 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4053 {
4054         u32 reset_mask;
4055
4056         if (hard) {
4057                 evergreen_gpu_pci_config_reset(rdev);
4058                 return 0;
4059         }
4060
4061         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4062
4063         if (reset_mask)
4064                 r600_set_bios_scratch_engine_hung(rdev, true);
4065
4066         /* try soft reset */
4067         evergreen_gpu_soft_reset(rdev, reset_mask);
4068
4069         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4070
4071         /* try pci config reset */
4072         if (reset_mask && radeon_hard_reset)
4073                 evergreen_gpu_pci_config_reset(rdev);
4074
4075         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4076
4077         if (!reset_mask)
4078                 r600_set_bios_scratch_engine_hung(rdev, false);
4079
4080         return 0;
4081 }
4082
4083 /**
4084  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4085  *
4086  * @rdev: radeon_device pointer
4087  * @ring: radeon_ring structure holding ring information
4088  *
4089  * Check if the GFX engine is locked up.
4090  * Returns true if the engine appears to be locked up, false if not.
4091  */
4092 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4093 {
4094         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4095
4096         if (!(reset_mask & (RADEON_RESET_GFX |
4097                             RADEON_RESET_COMPUTE |
4098                             RADEON_RESET_CP))) {
4099                 radeon_ring_lockup_update(rdev, ring);
4100                 return false;
4101         }
4102         return radeon_ring_test_lockup(rdev, ring);
4103 }
4104
4105 /*
4106  * RLC
4107  */
4108 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4109 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
4110
4111 void sumo_rlc_fini(struct radeon_device *rdev)
4112 {
4113         int r;
4114
4115         /* save restore block */
4116         if (rdev->rlc.save_restore_obj) {
4117                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4118                 if (unlikely(r != 0))
4119                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4120                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4121                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4122
4123                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4124                 rdev->rlc.save_restore_obj = NULL;
4125         }
4126
4127         /* clear state block */
4128         if (rdev->rlc.clear_state_obj) {
4129                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4130                 if (unlikely(r != 0))
4131                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4132                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4133                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4134
4135                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4136                 rdev->rlc.clear_state_obj = NULL;
4137         }
4138
4139         /* clear state block */
4140         if (rdev->rlc.cp_table_obj) {
4141                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4142                 if (unlikely(r != 0))
4143                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4144                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4145                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4146
4147                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4148                 rdev->rlc.cp_table_obj = NULL;
4149         }
4150 }
4151
4152 #define CP_ME_TABLE_SIZE    96
4153
4154 int sumo_rlc_init(struct radeon_device *rdev)
4155 {
4156         const u32 *src_ptr;
4157         volatile u32 *dst_ptr;
4158         u32 dws, data, i, j, k, reg_num;
4159         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4160         u64 reg_list_mc_addr;
4161         const struct cs_section_def *cs_data;
4162         int r;
4163
4164         src_ptr = rdev->rlc.reg_list;
4165         dws = rdev->rlc.reg_list_size;
4166         if (rdev->family >= CHIP_BONAIRE) {
4167                 dws += (5 * 16) + 48 + 48 + 64;
4168         }
4169         cs_data = rdev->rlc.cs_data;
4170
4171         if (src_ptr) {
4172                 /* save restore block */
4173                 if (rdev->rlc.save_restore_obj == NULL) {
4174                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4175                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4176                                              NULL, &rdev->rlc.save_restore_obj);
4177                         if (r) {
4178                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4179                                 return r;
4180                         }
4181                 }
4182
4183                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4184                 if (unlikely(r != 0)) {
4185                         sumo_rlc_fini(rdev);
4186                         return r;
4187                 }
4188                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4189                                   &rdev->rlc.save_restore_gpu_addr);
4190                 if (r) {
4191                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4192                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4193                         sumo_rlc_fini(rdev);
4194                         return r;
4195                 }
4196
4197                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4198                 if (r) {
4199                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4200                         sumo_rlc_fini(rdev);
4201                         return r;
4202                 }
4203                 /* write the sr buffer */
4204                 dst_ptr = rdev->rlc.sr_ptr;
4205                 if (rdev->family >= CHIP_TAHITI) {
4206                         /* SI */
4207                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4208                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4209                 } else {
4210                         /* ON/LN/TN */
4211                         /* format:
4212                          * dw0: (reg2 << 16) | reg1
4213                          * dw1: reg1 save space
4214                          * dw2: reg2 save space
4215                          */
4216                         for (i = 0; i < dws; i++) {
4217                                 data = src_ptr[i] >> 2;
4218                                 i++;
4219                                 if (i < dws)
4220                                         data |= (src_ptr[i] >> 2) << 16;
4221                                 j = (((i - 1) * 3) / 2);
4222                                 dst_ptr[j] = cpu_to_le32(data);
4223                         }
4224                         j = ((i * 3) / 2);
4225                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4226                 }
4227                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4228                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4229         }
4230
4231         if (cs_data) {
4232                 /* clear state block */
4233                 if (rdev->family >= CHIP_BONAIRE) {
4234                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4235                 } else if (rdev->family >= CHIP_TAHITI) {
4236                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4237                         dws = rdev->rlc.clear_state_size + (256 / 4);
4238                 } else {
4239                         reg_list_num = 0;
4240                         dws = 0;
4241                         for (i = 0; cs_data[i].section != NULL; i++) {
4242                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4243                                         reg_list_num++;
4244                                         dws += cs_data[i].section[j].reg_count;
4245                                 }
4246                         }
4247                         reg_list_blk_index = (3 * reg_list_num + 2);
4248                         dws += reg_list_blk_index;
4249                         rdev->rlc.clear_state_size = dws;
4250                 }
4251
4252                 if (rdev->rlc.clear_state_obj == NULL) {
4253                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4254                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4255                                              NULL, &rdev->rlc.clear_state_obj);
4256                         if (r) {
4257                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4258                                 sumo_rlc_fini(rdev);
4259                                 return r;
4260                         }
4261                 }
4262                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4263                 if (unlikely(r != 0)) {
4264                         sumo_rlc_fini(rdev);
4265                         return r;
4266                 }
4267                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4268                                   &rdev->rlc.clear_state_gpu_addr);
4269                 if (r) {
4270                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4271                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4272                         sumo_rlc_fini(rdev);
4273                         return r;
4274                 }
4275
4276                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4277                 if (r) {
4278                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4279                         sumo_rlc_fini(rdev);
4280                         return r;
4281                 }
4282                 /* set up the cs buffer */
4283                 dst_ptr = rdev->rlc.cs_ptr;
4284                 if (rdev->family >= CHIP_BONAIRE) {
4285                         cik_get_csb_buffer(rdev, dst_ptr);
4286                 } else if (rdev->family >= CHIP_TAHITI) {
4287                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4288                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4289                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4290                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4291                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4292                 } else {
4293                         reg_list_hdr_blk_index = 0;
4294                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4295                         data = upper_32_bits(reg_list_mc_addr);
4296                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4297                         reg_list_hdr_blk_index++;
4298                         for (i = 0; cs_data[i].section != NULL; i++) {
4299                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4300                                         reg_num = cs_data[i].section[j].reg_count;
4301                                         data = reg_list_mc_addr & 0xffffffff;
4302                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4303                                         reg_list_hdr_blk_index++;
4304
4305                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4306                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4307                                         reg_list_hdr_blk_index++;
4308
4309                                         data = 0x08000000 | (reg_num * 4);
4310                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4311                                         reg_list_hdr_blk_index++;
4312
4313                                         for (k = 0; k < reg_num; k++) {
4314                                                 data = cs_data[i].section[j].extent[k];
4315                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4316                                         }
4317                                         reg_list_mc_addr += reg_num * 4;
4318                                         reg_list_blk_index += reg_num;
4319                                 }
4320                         }
4321                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4322                 }
4323                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4324                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4325         }
4326
4327         if (rdev->rlc.cp_table_size) {
4328                 if (rdev->rlc.cp_table_obj == NULL) {
4329                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4330                                              PAGE_SIZE, true,
4331                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4332                                              NULL, &rdev->rlc.cp_table_obj);
4333                         if (r) {
4334                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4335                                 sumo_rlc_fini(rdev);
4336                                 return r;
4337                         }
4338                 }
4339
4340                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4341                 if (unlikely(r != 0)) {
4342                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4343                         sumo_rlc_fini(rdev);
4344                         return r;
4345                 }
4346                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4347                                   &rdev->rlc.cp_table_gpu_addr);
4348                 if (r) {
4349                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4350                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4351                         sumo_rlc_fini(rdev);
4352                         return r;
4353                 }
4354                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4355                 if (r) {
4356                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4357                         sumo_rlc_fini(rdev);
4358                         return r;
4359                 }
4360
4361                 cik_init_cp_pg_table(rdev);
4362
4363                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4364                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4365
4366         }
4367
4368         return 0;
4369 }
4370
4371 static void evergreen_rlc_start(struct radeon_device *rdev)
4372 {
4373         u32 mask = RLC_ENABLE;
4374
4375         if (rdev->flags & RADEON_IS_IGP) {
4376                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4377         }
4378
4379         WREG32(RLC_CNTL, mask);
4380 }
4381
4382 int evergreen_rlc_resume(struct radeon_device *rdev)
4383 {
4384         u32 i;
4385         const __be32 *fw_data;
4386
4387         if (!rdev->rlc_fw)
4388                 return -EINVAL;
4389
4390         r600_rlc_stop(rdev);
4391
4392         WREG32(RLC_HB_CNTL, 0);
4393
4394         if (rdev->flags & RADEON_IS_IGP) {
4395                 if (rdev->family == CHIP_ARUBA) {
4396                         u32 always_on_bitmap =
4397                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4398                         /* find out the number of active simds */
4399                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4400                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4401                         tmp = hweight32(~tmp);
4402                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4403                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4404                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4405                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4406                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4407                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4408                         }
4409                 } else {
4410                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4411                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4412                 }
4413                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4414                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4415         } else {
4416                 WREG32(RLC_HB_BASE, 0);
4417                 WREG32(RLC_HB_RPTR, 0);
4418                 WREG32(RLC_HB_WPTR, 0);
4419                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4420                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4421         }
4422         WREG32(RLC_MC_CNTL, 0);
4423         WREG32(RLC_UCODE_CNTL, 0);
4424
4425         fw_data = (const __be32 *)rdev->rlc_fw->data;
4426         if (rdev->family >= CHIP_ARUBA) {
4427                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4428                         WREG32(RLC_UCODE_ADDR, i);
4429                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4430                 }
4431         } else if (rdev->family >= CHIP_CAYMAN) {
4432                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4433                         WREG32(RLC_UCODE_ADDR, i);
4434                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4435                 }
4436         } else {
4437                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4438                         WREG32(RLC_UCODE_ADDR, i);
4439                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4440                 }
4441         }
4442         WREG32(RLC_UCODE_ADDR, 0);
4443
4444         evergreen_rlc_start(rdev);
4445
4446         return 0;
4447 }
4448
4449 /* Interrupts */
4450
4451 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4452 {
4453         if (crtc >= rdev->num_crtc)
4454                 return 0;
4455         else
4456                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4457 }
4458
4459 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4460 {
4461         int i;
4462         u32 tmp;
4463
4464         if (rdev->family >= CHIP_CAYMAN) {
4465                 cayman_cp_int_cntl_setup(rdev, 0,
4466                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4467                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4468                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4469                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4470                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4471         } else
4472                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4473         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4474         WREG32(DMA_CNTL, tmp);
4475         WREG32(GRBM_INT_CNTL, 0);
4476         WREG32(SRBM_INT_CNTL, 0);
4477         for (i = 0; i < rdev->num_crtc; i++)
4478                 WREG32(INT_MASK + crtc_offsets[i], 0);
4479         for (i = 0; i < rdev->num_crtc; i++)
4480                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4481
4482         /* only one DAC on DCE5 */
4483         if (!ASIC_IS_DCE5(rdev))
4484                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4485         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4486
4487         for (i = 0; i < 6; i++)
4488                 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4489 }
4490
4491 /* Note that the order we write back regs here is important */
4492 int evergreen_irq_set(struct radeon_device *rdev)
4493 {
4494         int i;
4495         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4496         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4497         u32 grbm_int_cntl = 0;
4498         u32 dma_cntl, dma_cntl1 = 0;
4499         u32 thermal_int = 0;
4500
4501         if (!rdev->irq.installed) {
4502                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4503                 return -EINVAL;
4504         }
4505         /* don't enable anything if the ih is disabled */
4506         if (!rdev->ih.enabled) {
4507                 r600_disable_interrupts(rdev);
4508                 /* force the active interrupt state to all disabled */
4509                 evergreen_disable_interrupt_state(rdev);
4510                 return 0;
4511         }
4512
4513         if (rdev->family == CHIP_ARUBA)
4514                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4515                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4516         else
4517                 thermal_int = RREG32(CG_THERMAL_INT) &
4518                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4519
4520         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4521
4522         if (rdev->family >= CHIP_CAYMAN) {
4523                 /* enable CP interrupts on all rings */
4524                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4525                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4526                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4527                 }
4528                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4529                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4530                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4531                 }
4532                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4533                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4534                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4535                 }
4536         } else {
4537                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4538                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4539                         cp_int_cntl |= RB_INT_ENABLE;
4540                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4541                 }
4542         }
4543
4544         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4545                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4546                 dma_cntl |= TRAP_ENABLE;
4547         }
4548
4549         if (rdev->family >= CHIP_CAYMAN) {
4550                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4551                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4552                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4553                         dma_cntl1 |= TRAP_ENABLE;
4554                 }
4555         }
4556
4557         if (rdev->irq.dpm_thermal) {
4558                 DRM_DEBUG("dpm thermal\n");
4559                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4560         }
4561
4562         if (rdev->family >= CHIP_CAYMAN) {
4563                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4564                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4565                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4566         } else
4567                 WREG32(CP_INT_CNTL, cp_int_cntl);
4568
4569         WREG32(DMA_CNTL, dma_cntl);
4570
4571         if (rdev->family >= CHIP_CAYMAN)
4572                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4573
4574         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4575
4576         for (i = 0; i < rdev->num_crtc; i++) {
4577                 radeon_irq_kms_set_irq_n_enabled(
4578                     rdev, INT_MASK + crtc_offsets[i],
4579                     VBLANK_INT_MASK,
4580                     rdev->irq.crtc_vblank_int[i] ||
4581                     atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4582         }
4583
4584         for (i = 0; i < rdev->num_crtc; i++)
4585                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4586
4587         for (i = 0; i < 6; i++) {
4588                 radeon_irq_kms_set_irq_n_enabled(
4589                     rdev, DC_HPDx_INT_CONTROL(i),
4590                     DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4591                     rdev->irq.hpd[i], "HPD", i);
4592         }
4593
4594         if (rdev->family == CHIP_ARUBA)
4595                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4596         else
4597                 WREG32(CG_THERMAL_INT, thermal_int);
4598
4599         for (i = 0; i < 6; i++) {
4600                 radeon_irq_kms_set_irq_n_enabled(
4601                     rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4602                     AFMT_AZ_FORMAT_WTRIG_MASK,
4603                     rdev->irq.afmt[i], "HDMI", i);
4604         }
4605
4606         /* posting read */
4607         RREG32(SRBM_STATUS);
4608
4609         return 0;
4610 }
4611
4612 /* Note that the order we write back regs here is important */
4613 static void evergreen_irq_ack(struct radeon_device *rdev)
4614 {
4615         int i, j;
4616         u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4617         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4618         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4619
4620         for (i = 0; i < 6; i++) {
4621                 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4622                 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4623                 if (i < rdev->num_crtc)
4624                         grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4625         }
4626
4627         /* We write back each interrupt register in pairs of two */
4628         for (i = 0; i < rdev->num_crtc; i += 2) {
4629                 for (j = i; j < (i + 2); j++) {
4630                         if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4631                                 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4632                                        GRPH_PFLIP_INT_CLEAR);
4633                 }
4634
4635                 for (j = i; j < (i + 2); j++) {
4636                         if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4637                                 WREG32(VBLANK_STATUS + crtc_offsets[j],
4638                                        VBLANK_ACK);
4639                         if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4640                                 WREG32(VLINE_STATUS + crtc_offsets[j],
4641                                        VLINE_ACK);
4642                 }
4643         }
4644
4645         for (i = 0; i < 6; i++) {
4646                 if (disp_int[i] & DC_HPD1_INTERRUPT)
4647                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4648         }
4649
4650         for (i = 0; i < 6; i++) {
4651                 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4652                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4653         }
4654
4655         for (i = 0; i < 6; i++) {
4656                 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4657                         WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4658                                   AFMT_AZ_FORMAT_WTRIG_ACK);
4659         }
4660 }
4661
4662 static void evergreen_irq_disable(struct radeon_device *rdev)
4663 {
4664         r600_disable_interrupts(rdev);
4665         /* Wait and acknowledge irq */
4666         mdelay(1);
4667         evergreen_irq_ack(rdev);
4668         evergreen_disable_interrupt_state(rdev);
4669 }
4670
4671 void evergreen_irq_suspend(struct radeon_device *rdev)
4672 {
4673         evergreen_irq_disable(rdev);
4674         r600_rlc_stop(rdev);
4675 }
4676
4677 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4678 {
4679         u32 wptr, tmp;
4680
4681         if (rdev->wb.enabled)
4682                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4683         else
4684                 wptr = RREG32(IH_RB_WPTR);
4685
4686         if (wptr & RB_OVERFLOW) {
4687                 wptr &= ~RB_OVERFLOW;
4688                 /* When a ring buffer overflow happen start parsing interrupt
4689                  * from the last not overwritten vector (wptr + 16). Hopefully
4690                  * this should allow us to catchup.
4691                  */
4692                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4693                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4694                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4695                 tmp = RREG32(IH_RB_CNTL);
4696                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4697                 WREG32(IH_RB_CNTL, tmp);
4698         }
4699         return (wptr & rdev->ih.ptr_mask);
4700 }
4701
4702 int evergreen_irq_process(struct radeon_device *rdev)
4703 {
4704         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4705         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4706         u32 crtc_idx, hpd_idx, afmt_idx;
4707         u32 mask;
4708         u32 wptr;
4709         u32 rptr;
4710         u32 src_id, src_data;
4711         u32 ring_index;
4712         bool queue_hotplug = false;
4713         bool queue_hdmi = false;
4714         bool queue_dp = false;
4715         bool queue_thermal = false;
4716         u32 status, addr;
4717         const char *event_name;
4718
4719         if (!rdev->ih.enabled || rdev->shutdown)
4720                 return IRQ_NONE;
4721
4722         wptr = evergreen_get_ih_wptr(rdev);
4723
4724 restart_ih:
4725         /* is somebody else already processing irqs? */
4726         if (atomic_xchg(&rdev->ih.lock, 1))
4727                 return IRQ_NONE;
4728
4729         rptr = rdev->ih.rptr;
4730         DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4731
4732         /* Order reading of wptr vs. reading of IH ring data */
4733         rmb();
4734
4735         /* display interrupts */
4736         evergreen_irq_ack(rdev);
4737
4738         while (rptr != wptr) {
4739                 /* wptr/rptr are in bytes! */
4740                 ring_index = rptr / 4;
4741                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4742                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4743
4744                 switch (src_id) {
4745                 case 1: /* D1 vblank/vline */
4746                 case 2: /* D2 vblank/vline */
4747                 case 3: /* D3 vblank/vline */
4748                 case 4: /* D4 vblank/vline */
4749                 case 5: /* D5 vblank/vline */
4750                 case 6: /* D6 vblank/vline */
4751                         crtc_idx = src_id - 1;
4752
4753                         if (src_data == 0) { /* vblank */
4754                                 mask = LB_D1_VBLANK_INTERRUPT;
4755                                 event_name = "vblank";
4756
4757                                 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4758                                         drm_handle_vblank(rdev->ddev, crtc_idx);
4759                                         rdev->pm.vblank_sync = true;
4760                                         wake_up(&rdev->irq.vblank_queue);
4761                                 }
4762                                 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4763                                         radeon_crtc_handle_vblank(rdev,
4764                                                                   crtc_idx);
4765                                 }
4766
4767                         } else if (src_data == 1) { /* vline */
4768                                 mask = LB_D1_VLINE_INTERRUPT;
4769                                 event_name = "vline";
4770                         } else {
4771                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4772                                           src_id, src_data);
4773                                 break;
4774                         }
4775
4776                         if (!(disp_int[crtc_idx] & mask)) {
4777                                 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4778                                           crtc_idx + 1, event_name);
4779                         }
4780
4781                         disp_int[crtc_idx] &= ~mask;
4782                         DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4783
4784                         break;
4785                 case 8: /* D1 page flip */
4786                 case 10: /* D2 page flip */
4787                 case 12: /* D3 page flip */
4788                 case 14: /* D4 page flip */
4789                 case 16: /* D5 page flip */
4790                 case 18: /* D6 page flip */
4791                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4792                         if (radeon_use_pflipirq > 0)
4793                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4794                         break;
4795                 case 42: /* HPD hotplug */
4796                         if (src_data <= 5) {
4797                                 hpd_idx = src_data;
4798                                 mask = DC_HPD1_INTERRUPT;
4799                                 queue_hotplug = true;
4800                                 event_name = "HPD";
4801
4802                         } else if (src_data <= 11) {
4803                                 hpd_idx = src_data - 6;
4804                                 mask = DC_HPD1_RX_INTERRUPT;
4805                                 queue_dp = true;
4806                                 event_name = "HPD_RX";
4807
4808                         } else {
4809                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4810                                           src_id, src_data);
4811                                 break;
4812                         }
4813
4814                         if (!(disp_int[hpd_idx] & mask))
4815                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4816
4817                         disp_int[hpd_idx] &= ~mask;
4818                         DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4819
4820                         break;
4821                 case 44: /* hdmi */
4822                         afmt_idx = src_data;
4823                         if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4824                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4825
4826                         if (afmt_idx > 5) {
4827                                 DRM_ERROR("Unhandled interrupt: %d %d\n",
4828                                           src_id, src_data);
4829                                 break;
4830                         }
4831                         afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4832                         queue_hdmi = true;
4833                         DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4834                         break;
4835                 case 96:
4836                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4837                         WREG32(SRBM_INT_ACK, 0x1);
4838                         break;
4839                 case 124: /* UVD */
4840                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4841                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4842                         break;
4843                 case 146:
4844                 case 147:
4845                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4846                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4847                         /* reset addr and status */
4848                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4849                         if (addr == 0x0 && status == 0x0)
4850                                 break;
4851                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4852                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4853                                 addr);
4854                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4855                                 status);
4856                         cayman_vm_decode_fault(rdev, status, addr);
4857                         break;
4858                 case 176: /* CP_INT in ring buffer */
4859                 case 177: /* CP_INT in IB1 */
4860                 case 178: /* CP_INT in IB2 */
4861                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4862                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4863                         break;
4864                 case 181: /* CP EOP event */
4865                         DRM_DEBUG("IH: CP EOP\n");
4866                         if (rdev->family >= CHIP_CAYMAN) {
4867                                 switch (src_data) {
4868                                 case 0:
4869                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4870                                         break;
4871                                 case 1:
4872                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4873                                         break;
4874                                 case 2:
4875                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4876                                         break;
4877                                 }
4878                         } else
4879                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4880                         break;
4881                 case 224: /* DMA trap event */
4882                         DRM_DEBUG("IH: DMA trap\n");
4883                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4884                         break;
4885                 case 230: /* thermal low to high */
4886                         DRM_DEBUG("IH: thermal low to high\n");
4887                         rdev->pm.dpm.thermal.high_to_low = false;
4888                         queue_thermal = true;
4889                         break;
4890                 case 231: /* thermal high to low */
4891                         DRM_DEBUG("IH: thermal high to low\n");
4892                         rdev->pm.dpm.thermal.high_to_low = true;
4893                         queue_thermal = true;
4894                         break;
4895                 case 233: /* GUI IDLE */
4896                         DRM_DEBUG("IH: GUI idle\n");
4897                         break;
4898                 case 244: /* DMA trap event */
4899                         if (rdev->family >= CHIP_CAYMAN) {
4900                                 DRM_DEBUG("IH: DMA1 trap\n");
4901                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4902                         }
4903                         break;
4904                 default:
4905                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4906                         break;
4907                 }
4908
4909                 /* wptr/rptr are in bytes! */
4910                 rptr += 16;
4911                 rptr &= rdev->ih.ptr_mask;
4912                 WREG32(IH_RB_RPTR, rptr);
4913         }
4914         if (queue_dp)
4915                 schedule_work(&rdev->dp_work);
4916         if (queue_hotplug)
4917                 schedule_delayed_work(&rdev->hotplug_work, 0);
4918         if (queue_hdmi)
4919                 schedule_work(&rdev->audio_work);
4920         if (queue_thermal && rdev->pm.dpm_enabled)
4921                 schedule_work(&rdev->pm.dpm.thermal.work);
4922         rdev->ih.rptr = rptr;
4923         atomic_set(&rdev->ih.lock, 0);
4924
4925         /* make sure wptr hasn't changed while processing */
4926         wptr = evergreen_get_ih_wptr(rdev);
4927         if (wptr != rptr)
4928                 goto restart_ih;
4929
4930         return IRQ_HANDLED;
4931 }
4932
4933 static void evergreen_uvd_init(struct radeon_device *rdev)
4934 {
4935         int r;
4936
4937         if (!rdev->has_uvd)
4938                 return;
4939
4940         r = radeon_uvd_init(rdev);
4941         if (r) {
4942                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4943                 /*
4944                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4945                  * to early fails uvd_v2_2_resume() and thus nothing happens
4946                  * there. So it is pointless to try to go through that code
4947                  * hence why we disable uvd here.
4948                  */
4949                 rdev->has_uvd = false;
4950                 return;
4951         }
4952         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4953         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4954 }
4955
4956 static void evergreen_uvd_start(struct radeon_device *rdev)
4957 {
4958         int r;
4959
4960         if (!rdev->has_uvd)
4961                 return;
4962
4963         r = uvd_v2_2_resume(rdev);
4964         if (r) {
4965                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4966                 goto error;
4967         }
4968         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4969         if (r) {
4970                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4971                 goto error;
4972         }
4973         return;
4974
4975 error:
4976         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4977 }
4978
4979 static void evergreen_uvd_resume(struct radeon_device *rdev)
4980 {
4981         struct radeon_ring *ring;
4982         int r;
4983
4984         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4985                 return;
4986
4987         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4988         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4989         if (r) {
4990                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4991                 return;
4992         }
4993         r = uvd_v1_0_init(rdev);
4994         if (r) {
4995                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4996                 return;
4997         }
4998 }
4999
5000 static int evergreen_startup(struct radeon_device *rdev)
5001 {
5002         struct radeon_ring *ring;
5003         int r;
5004
5005         /* enable pcie gen2 link */
5006         evergreen_pcie_gen2_enable(rdev);
5007         /* enable aspm */
5008         evergreen_program_aspm(rdev);
5009
5010         /* scratch needs to be initialized before MC */
5011         r = r600_vram_scratch_init(rdev);
5012         if (r)
5013                 return r;
5014
5015         evergreen_mc_program(rdev);
5016
5017         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5018                 r = ni_mc_load_microcode(rdev);
5019                 if (r) {
5020                         DRM_ERROR("Failed to load MC firmware!\n");
5021                         return r;
5022                 }
5023         }
5024
5025         if (rdev->flags & RADEON_IS_AGP) {
5026                 evergreen_agp_enable(rdev);
5027         } else {
5028                 r = evergreen_pcie_gart_enable(rdev);
5029                 if (r)
5030                         return r;
5031         }
5032         evergreen_gpu_init(rdev);
5033
5034         /* allocate rlc buffers */
5035         if (rdev->flags & RADEON_IS_IGP) {
5036                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5037                 rdev->rlc.reg_list_size =
5038                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5039                 rdev->rlc.cs_data = evergreen_cs_data;
5040                 r = sumo_rlc_init(rdev);
5041                 if (r) {
5042                         DRM_ERROR("Failed to init rlc BOs!\n");
5043                         return r;
5044                 }
5045         }
5046
5047         /* allocate wb buffer */
5048         r = radeon_wb_init(rdev);
5049         if (r)
5050                 return r;
5051
5052         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5053         if (r) {
5054                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5055                 return r;
5056         }
5057
5058         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5059         if (r) {
5060                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5061                 return r;
5062         }
5063
5064         evergreen_uvd_start(rdev);
5065
5066         /* Enable IRQ */
5067         if (!rdev->irq.installed) {
5068                 r = radeon_irq_kms_init(rdev);
5069                 if (r)
5070                         return r;
5071         }
5072
5073         r = r600_irq_init(rdev);
5074         if (r) {
5075                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5076                 radeon_irq_kms_fini(rdev);
5077                 return r;
5078         }
5079         evergreen_irq_set(rdev);
5080
5081         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5082         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5083                              RADEON_CP_PACKET2);
5084         if (r)
5085                 return r;
5086
5087         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5088         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5089                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5090         if (r)
5091                 return r;
5092
5093         r = evergreen_cp_load_microcode(rdev);
5094         if (r)
5095                 return r;
5096         r = evergreen_cp_resume(rdev);
5097         if (r)
5098                 return r;
5099         r = r600_dma_resume(rdev);
5100         if (r)
5101                 return r;
5102
5103         evergreen_uvd_resume(rdev);
5104
5105         r = radeon_ib_pool_init(rdev);
5106         if (r) {
5107                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5108                 return r;
5109         }
5110
5111         r = radeon_audio_init(rdev);
5112         if (r) {
5113                 DRM_ERROR("radeon: audio init failed\n");
5114                 return r;
5115         }
5116
5117         return 0;
5118 }
5119
5120 int evergreen_resume(struct radeon_device *rdev)
5121 {
5122         int r;
5123
5124         /* reset the asic, the gfx blocks are often in a bad state
5125          * after the driver is unloaded or after a resume
5126          */
5127         if (radeon_asic_reset(rdev))
5128                 dev_warn(rdev->dev, "GPU reset failed !\n");
5129         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5130          * posting will perform necessary task to bring back GPU into good
5131          * shape.
5132          */
5133         /* post card */
5134         atom_asic_init(rdev->mode_info.atom_context);
5135
5136         /* init golden registers */
5137         evergreen_init_golden_registers(rdev);
5138
5139         if (rdev->pm.pm_method == PM_METHOD_DPM)
5140                 radeon_pm_resume(rdev);
5141
5142         rdev->accel_working = true;
5143         r = evergreen_startup(rdev);
5144         if (r) {
5145                 DRM_ERROR("evergreen startup failed on resume\n");
5146                 rdev->accel_working = false;
5147                 return r;
5148         }
5149
5150         return r;
5151
5152 }
5153
5154 int evergreen_suspend(struct radeon_device *rdev)
5155 {
5156         radeon_pm_suspend(rdev);
5157         radeon_audio_fini(rdev);
5158         if (rdev->has_uvd) {
5159                 radeon_uvd_suspend(rdev);
5160                 uvd_v1_0_fini(rdev);
5161         }
5162         r700_cp_stop(rdev);
5163         r600_dma_stop(rdev);
5164         evergreen_irq_suspend(rdev);
5165         radeon_wb_disable(rdev);
5166         evergreen_pcie_gart_disable(rdev);
5167
5168         return 0;
5169 }
5170
5171 /* Plan is to move initialization in that function and use
5172  * helper function so that radeon_device_init pretty much
5173  * do nothing more than calling asic specific function. This
5174  * should also allow to remove a bunch of callback function
5175  * like vram_info.
5176  */
5177 int evergreen_init(struct radeon_device *rdev)
5178 {
5179         int r;
5180
5181         /* Read BIOS */
5182         if (!radeon_get_bios(rdev)) {
5183                 if (ASIC_IS_AVIVO(rdev))
5184                         return -EINVAL;
5185         }
5186         /* Must be an ATOMBIOS */
5187         if (!rdev->is_atom_bios) {
5188                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5189                 return -EINVAL;
5190         }
5191         r = radeon_atombios_init(rdev);
5192         if (r)
5193                 return r;
5194         /* reset the asic, the gfx blocks are often in a bad state
5195          * after the driver is unloaded or after a resume
5196          */
5197         if (radeon_asic_reset(rdev))
5198                 dev_warn(rdev->dev, "GPU reset failed !\n");
5199         /* Post card if necessary */
5200         if (!radeon_card_posted(rdev)) {
5201                 if (!rdev->bios) {
5202                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5203                         return -EINVAL;
5204                 }
5205                 DRM_INFO("GPU not posted. posting now...\n");
5206                 atom_asic_init(rdev->mode_info.atom_context);
5207         }
5208         /* init golden registers */
5209         evergreen_init_golden_registers(rdev);
5210         /* Initialize scratch registers */
5211         r600_scratch_init(rdev);
5212         /* Initialize surface registers */
5213         radeon_surface_init(rdev);
5214         /* Initialize clocks */
5215         radeon_get_clock_info(rdev->ddev);
5216         /* Fence driver */
5217         radeon_fence_driver_init(rdev);
5218         /* initialize AGP */
5219         if (rdev->flags & RADEON_IS_AGP) {
5220                 r = radeon_agp_init(rdev);
5221                 if (r)
5222                         radeon_agp_disable(rdev);
5223         }
5224         /* initialize memory controller */
5225         r = evergreen_mc_init(rdev);
5226         if (r)
5227                 return r;
5228         /* Memory manager */
5229         r = radeon_bo_init(rdev);
5230         if (r)
5231                 return r;
5232
5233         if (ASIC_IS_DCE5(rdev)) {
5234                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5235                         r = ni_init_microcode(rdev);
5236                         if (r) {
5237                                 DRM_ERROR("Failed to load firmware!\n");
5238                                 return r;
5239                         }
5240                 }
5241         } else {
5242                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5243                         r = r600_init_microcode(rdev);
5244                         if (r) {
5245                                 DRM_ERROR("Failed to load firmware!\n");
5246                                 /*(DEBLOBBED)*/
5247                         }
5248                 }
5249         }
5250
5251         /* Initialize power management */
5252         radeon_pm_init(rdev);
5253
5254         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5255         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5256
5257         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5258         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5259
5260         evergreen_uvd_init(rdev);
5261
5262         rdev->ih.ring_obj = NULL;
5263         r600_ih_ring_init(rdev, 64 * 1024);
5264
5265         r = r600_pcie_gart_init(rdev);
5266         if (r)
5267                 return r;
5268
5269         rdev->accel_working = true;
5270         r = evergreen_startup(rdev);
5271         if (r) {
5272                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5273                 r700_cp_fini(rdev);
5274                 r600_dma_fini(rdev);
5275                 r600_irq_fini(rdev);
5276                 if (rdev->flags & RADEON_IS_IGP)
5277                         sumo_rlc_fini(rdev);
5278                 radeon_wb_fini(rdev);
5279                 radeon_ib_pool_fini(rdev);
5280                 radeon_irq_kms_fini(rdev);
5281                 evergreen_pcie_gart_fini(rdev);
5282                 rdev->accel_working = false;
5283         }
5284
5285         /* Don't start up if the MC ucode is missing on BTC parts.
5286          * The default clocks and voltages before the MC ucode
5287          * is loaded are not suffient for advanced operations.
5288          */
5289         if (ASIC_IS_DCE5(rdev)) {
5290                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5291                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5292                         return -EINVAL;
5293                 }
5294         }
5295
5296         return 0;
5297 }
5298
5299 void evergreen_fini(struct radeon_device *rdev)
5300 {
5301         radeon_pm_fini(rdev);
5302         radeon_audio_fini(rdev);
5303         r700_cp_fini(rdev);
5304         r600_dma_fini(rdev);
5305         r600_irq_fini(rdev);
5306         if (rdev->flags & RADEON_IS_IGP)
5307                 sumo_rlc_fini(rdev);
5308         radeon_wb_fini(rdev);
5309         radeon_ib_pool_fini(rdev);
5310         radeon_irq_kms_fini(rdev);
5311         uvd_v1_0_fini(rdev);
5312         radeon_uvd_fini(rdev);
5313         evergreen_pcie_gart_fini(rdev);
5314         r600_vram_scratch_fini(rdev);
5315         radeon_gem_fini(rdev);
5316         radeon_fence_driver_fini(rdev);
5317         radeon_agp_fini(rdev);
5318         radeon_bo_fini(rdev);
5319         radeon_atombios_fini(rdev);
5320         kfree(rdev->bios);
5321         rdev->bios = NULL;
5322 }
5323
5324 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5325 {
5326         u32 link_width_cntl, speed_cntl;
5327
5328         if (radeon_pcie_gen2 == 0)
5329                 return;
5330
5331         if (rdev->flags & RADEON_IS_IGP)
5332                 return;
5333
5334         if (!(rdev->flags & RADEON_IS_PCIE))
5335                 return;
5336
5337         /* x2 cards have a special sequence */
5338         if (ASIC_IS_X2(rdev))
5339                 return;
5340
5341         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5342                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5343                 return;
5344
5345         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5346         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5347                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5348                 return;
5349         }
5350
5351         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5352
5353         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5354             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5355
5356                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5357                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5358                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5359
5360                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5361                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5362                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5363
5364                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5365                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5366                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5367
5368                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5369                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5370                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5371
5372                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5373                 speed_cntl |= LC_GEN2_EN_STRAP;
5374                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5375
5376         } else {
5377                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5378                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5379                 if (1)
5380                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5381                 else
5382                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5383                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5384         }
5385 }
5386
5387 void evergreen_program_aspm(struct radeon_device *rdev)
5388 {
5389         u32 data, orig;
5390         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5391         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5392         /* fusion_platform = true
5393          * if the system is a fusion system
5394          * (APU or DGPU in a fusion system).
5395          * todo: check if the system is a fusion platform.
5396          */
5397         bool fusion_platform = false;
5398
5399         if (radeon_aspm == 0)
5400                 return;
5401
5402         if (!(rdev->flags & RADEON_IS_PCIE))
5403                 return;
5404
5405         switch (rdev->family) {
5406         case CHIP_CYPRESS:
5407         case CHIP_HEMLOCK:
5408         case CHIP_JUNIPER:
5409         case CHIP_REDWOOD:
5410         case CHIP_CEDAR:
5411         case CHIP_SUMO:
5412         case CHIP_SUMO2:
5413         case CHIP_PALM:
5414         case CHIP_ARUBA:
5415                 disable_l0s = true;
5416                 break;
5417         default:
5418                 disable_l0s = false;
5419                 break;
5420         }
5421
5422         if (rdev->flags & RADEON_IS_IGP)
5423                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5424
5425         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5426         if (fusion_platform)
5427                 data &= ~MULTI_PIF;
5428         else
5429                 data |= MULTI_PIF;
5430         if (data != orig)
5431                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5432
5433         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5434         if (fusion_platform)
5435                 data &= ~MULTI_PIF;
5436         else
5437                 data |= MULTI_PIF;
5438         if (data != orig)
5439                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5440
5441         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5442         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5443         if (!disable_l0s) {
5444                 if (rdev->family >= CHIP_BARTS)
5445                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5446                 else
5447                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5448         }
5449
5450         if (!disable_l1) {
5451                 if (rdev->family >= CHIP_BARTS)
5452                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5453                 else
5454                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5455
5456                 if (!disable_plloff_in_l1) {
5457                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5458                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5459                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5460                         if (data != orig)
5461                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5462
5463                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5464                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5465                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5466                         if (data != orig)
5467                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5468
5469                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5470                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5471                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5472                         if (data != orig)
5473                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5474
5475                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5476                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5477                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5478                         if (data != orig)
5479                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5480
5481                         if (rdev->family >= CHIP_BARTS) {
5482                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5483                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5484                                 data |= PLL_RAMP_UP_TIME_0(4);
5485                                 if (data != orig)
5486                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5487
5488                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5489                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5490                                 data |= PLL_RAMP_UP_TIME_1(4);
5491                                 if (data != orig)
5492                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5493
5494                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5495                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5496                                 data |= PLL_RAMP_UP_TIME_0(4);
5497                                 if (data != orig)
5498                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5499
5500                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5501                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5502                                 data |= PLL_RAMP_UP_TIME_1(4);
5503                                 if (data != orig)
5504                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5505                         }
5506
5507                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5508                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5509                         data |= LC_DYN_LANES_PWR_STATE(3);
5510                         if (data != orig)
5511                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5512
5513                         if (rdev->family >= CHIP_BARTS) {
5514                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5515                                 data &= ~LS2_EXIT_TIME_MASK;
5516                                 data |= LS2_EXIT_TIME(1);
5517                                 if (data != orig)
5518                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5519
5520                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5521                                 data &= ~LS2_EXIT_TIME_MASK;
5522                                 data |= LS2_EXIT_TIME(1);
5523                                 if (data != orig)
5524                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5525                         }
5526                 }
5527         }
5528
5529         /* evergreen parts only */
5530         if (rdev->family < CHIP_BARTS)
5531                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5532
5533         if (pcie_lc_cntl != pcie_lc_cntl_old)
5534                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5535 }