drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/pci.h>
  27 #include <linux/slab.h>
  28
  29 #include <drm/drm_vblank.h>
  30 #include <drm/radeon_drm.h>
  31
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_blit_shaders.h"
  35 #include "evergreen_reg.h"
  36 #include "evergreend.h"
  37 #include "radeon.h"
  38 #include "radeon_asic.h"
  39 #include "radeon_audio.h"
  40 #include "radeon_ucode.h"
  41
  42 #define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  43 #define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  44 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  45
  46 /*
  47  * Indirect registers accessor
  48  */
  49 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  50 {
  51         unsigned long flags;
  52         u32 r;
  53
  54         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  55         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  56         r = RREG32(EVERGREEN_CG_IND_DATA);
  57         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  58         return r;
  59 }
  60
  61 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  62 {
  63         unsigned long flags;
  64
  65         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  66         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  67         WREG32(EVERGREEN_CG_IND_DATA, (v));
  68         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  69 }
  70
  71 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  72 {
  73         unsigned long flags;
  74         u32 r;
  75
  76         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  77         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  78         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  79         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  80         return r;
  81 }
  82
  83 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  84 {
  85         unsigned long flags;
  86
  87         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  88         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  89         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  90         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  91 }
  92
  93 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  94 {
  95         unsigned long flags;
  96         u32 r;
  97
  98         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  99         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 100         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 101         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 102         return r;
 103 }
 104
 105 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 106 {
 107         unsigned long flags;
 108
 109         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 110         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 111         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 112         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 113 }
 114
 115 static const u32 crtc_offsets[6] =
 116 {
 117         EVERGREEN_CRTC0_REGISTER_OFFSET,
 118         EVERGREEN_CRTC1_REGISTER_OFFSET,
 119         EVERGREEN_CRTC2_REGISTER_OFFSET,
 120         EVERGREEN_CRTC3_REGISTER_OFFSET,
 121         EVERGREEN_CRTC4_REGISTER_OFFSET,
 122         EVERGREEN_CRTC5_REGISTER_OFFSET
 123 };
 124
 125 #include "clearstate_evergreen.h"
 126
 127 static const u32 sumo_rlc_save_restore_register_list[] =
 128 {
 129         0x98fc,
 130         0x9830,
 131         0x9834,
 132         0x9838,
 133         0x9870,
 134         0x9874,
 135         0x8a14,
 136         0x8b24,
 137         0x8bcc,
 138         0x8b10,
 139         0x8d00,
 140         0x8d04,
 141         0x8c00,
 142         0x8c04,
 143         0x8c08,
 144         0x8c0c,
 145         0x8d8c,
 146         0x8c20,
 147         0x8c24,
 148         0x8c28,
 149         0x8c18,
 150         0x8c1c,
 151         0x8cf0,
 152         0x8e2c,
 153         0x8e38,
 154         0x8c30,
 155         0x9508,
 156         0x9688,
 157         0x9608,
 158         0x960c,
 159         0x9610,
 160         0x9614,
 161         0x88c4,
 162         0x88d4,
 163         0xa008,
 164         0x900c,
 165         0x9100,
 166         0x913c,
 167         0x98f8,
 168         0x98f4,
 169         0x9b7c,
 170         0x3f8c,
 171         0x8950,
 172         0x8954,
 173         0x8a18,
 174         0x8b28,
 175         0x9144,
 176         0x9148,
 177         0x914c,
 178         0x3f90,
 179         0x3f94,
 180         0x915c,
 181         0x9160,
 182         0x9178,
 183         0x917c,
 184         0x9180,
 185         0x918c,
 186         0x9190,
 187         0x9194,
 188         0x9198,
 189         0x919c,
 190         0x91a8,
 191         0x91ac,
 192         0x91b0,
 193         0x91b4,
 194         0x91b8,
 195         0x91c4,
 196         0x91c8,
 197         0x91cc,
 198         0x91d0,
 199         0x91d4,
 200         0x91e0,
 201         0x91e4,
 202         0x91ec,
 203         0x91f0,
 204         0x91f4,
 205         0x9200,
 206         0x9204,
 207         0x929c,
 208         0x9150,
 209         0x802c,
 210 };
 211
 212 static void evergreen_gpu_init(struct radeon_device *rdev);
 213 void evergreen_fini(struct radeon_device *rdev);
 214 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 215 void evergreen_program_aspm(struct radeon_device *rdev);
 216 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 217                                      int ring, u32 cp_int_cntl);
 218 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 219                                    u32 status, u32 addr);
 220 void cik_init_cp_pg_table(struct radeon_device *rdev);
 221
 222 extern u32 si_get_csb_size(struct radeon_device *rdev);
 223 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 224 extern u32 cik_get_csb_size(struct radeon_device *rdev);
 225 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 226 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 227
 228 static const u32 evergreen_golden_registers[] =
 229 {
 230         0x3f90, 0xffff0000, 0xff000000,
 231         0x9148, 0xffff0000, 0xff000000,
 232         0x3f94, 0xffff0000, 0xff000000,
 233         0x914c, 0xffff0000, 0xff000000,
 234         0x9b7c, 0xffffffff, 0x00000000,
 235         0x8a14, 0xffffffff, 0x00000007,
 236         0x8b10, 0xffffffff, 0x00000000,
 237         0x960c, 0xffffffff, 0x54763210,
 238         0x88c4, 0xffffffff, 0x000000c2,
 239         0x88d4, 0xffffffff, 0x00000010,
 240         0x8974, 0xffffffff, 0x00000000,
 241         0xc78, 0x00000080, 0x00000080,
 242         0x5eb4, 0xffffffff, 0x00000002,
 243         0x5e78, 0xffffffff, 0x001000f0,
 244         0x6104, 0x01000300, 0x00000000,
 245         0x5bc0, 0x00300000, 0x00000000,
 246         0x7030, 0xffffffff, 0x00000011,
 247         0x7c30, 0xffffffff, 0x00000011,
 248         0x10830, 0xffffffff, 0x00000011,
 249         0x11430, 0xffffffff, 0x00000011,
 250         0x12030, 0xffffffff, 0x00000011,
 251         0x12c30, 0xffffffff, 0x00000011,
 252         0xd02c, 0xffffffff, 0x08421000,
 253         0x240c, 0xffffffff, 0x00000380,
 254         0x8b24, 0xffffffff, 0x00ff0fff,
 255         0x28a4c, 0x06000000, 0x06000000,
 256         0x10c, 0x00000001, 0x00000001,
 257         0x8d00, 0xffffffff, 0x100e4848,
 258         0x8d04, 0xffffffff, 0x00164745,
 259         0x8c00, 0xffffffff, 0xe4000003,
 260         0x8c04, 0xffffffff, 0x40600060,
 261         0x8c08, 0xffffffff, 0x001c001c,
 262         0x8cf0, 0xffffffff, 0x08e00620,
 263         0x8c20, 0xffffffff, 0x00800080,
 264         0x8c24, 0xffffffff, 0x00800080,
 265         0x8c18, 0xffffffff, 0x20202078,
 266         0x8c1c, 0xffffffff, 0x00001010,
 267         0x28350, 0xffffffff, 0x00000000,
 268         0xa008, 0xffffffff, 0x00010000,
 269         0x5c4, 0xffffffff, 0x00000001,
 270         0x9508, 0xffffffff, 0x00000002,
 271         0x913c, 0x0000000f, 0x0000000a
 272 };
 273
 274 static const u32 evergreen_golden_registers2[] =
 275 {
 276         0x2f4c, 0xffffffff, 0x00000000,
 277         0x54f4, 0xffffffff, 0x00000000,
 278         0x54f0, 0xffffffff, 0x00000000,
 279         0x5498, 0xffffffff, 0x00000000,
 280         0x549c, 0xffffffff, 0x00000000,
 281         0x5494, 0xffffffff, 0x00000000,
 282         0x53cc, 0xffffffff, 0x00000000,
 283         0x53c8, 0xffffffff, 0x00000000,
 284         0x53c4, 0xffffffff, 0x00000000,
 285         0x53c0, 0xffffffff, 0x00000000,
 286         0x53bc, 0xffffffff, 0x00000000,
 287         0x53b8, 0xffffffff, 0x00000000,
 288         0x53b4, 0xffffffff, 0x00000000,
 289         0x53b0, 0xffffffff, 0x00000000
 290 };
 291
 292 static const u32 cypress_mgcg_init[] =
 293 {
 294         0x802c, 0xffffffff, 0xc0000000,
 295         0x5448, 0xffffffff, 0x00000100,
 296         0x55e4, 0xffffffff, 0x00000100,
 297         0x160c, 0xffffffff, 0x00000100,
 298         0x5644, 0xffffffff, 0x00000100,
 299         0xc164, 0xffffffff, 0x00000100,
 300         0x8a18, 0xffffffff, 0x00000100,
 301         0x897c, 0xffffffff, 0x06000100,
 302         0x8b28, 0xffffffff, 0x00000100,
 303         0x9144, 0xffffffff, 0x00000100,
 304         0x9a60, 0xffffffff, 0x00000100,
 305         0x9868, 0xffffffff, 0x00000100,
 306         0x8d58, 0xffffffff, 0x00000100,
 307         0x9510, 0xffffffff, 0x00000100,
 308         0x949c, 0xffffffff, 0x00000100,
 309         0x9654, 0xffffffff, 0x00000100,
 310         0x9030, 0xffffffff, 0x00000100,
 311         0x9034, 0xffffffff, 0x00000100,
 312         0x9038, 0xffffffff, 0x00000100,
 313         0x903c, 0xffffffff, 0x00000100,
 314         0x9040, 0xffffffff, 0x00000100,
 315         0xa200, 0xffffffff, 0x00000100,
 316         0xa204, 0xffffffff, 0x00000100,
 317         0xa208, 0xffffffff, 0x00000100,
 318         0xa20c, 0xffffffff, 0x00000100,
 319         0x971c, 0xffffffff, 0x00000100,
 320         0x977c, 0xffffffff, 0x00000100,
 321         0x3f80, 0xffffffff, 0x00000100,
 322         0xa210, 0xffffffff, 0x00000100,
 323         0xa214, 0xffffffff, 0x00000100,
 324         0x4d8, 0xffffffff, 0x00000100,
 325         0x9784, 0xffffffff, 0x00000100,
 326         0x9698, 0xffffffff, 0x00000100,
 327         0x4d4, 0xffffffff, 0x00000200,
 328         0x30cc, 0xffffffff, 0x00000100,
 329         0xd0c0, 0xffffffff, 0xff000100,
 330         0x802c, 0xffffffff, 0x40000000,
 331         0x915c, 0xffffffff, 0x00010000,
 332         0x9160, 0xffffffff, 0x00030002,
 333         0x9178, 0xffffffff, 0x00070000,
 334         0x917c, 0xffffffff, 0x00030002,
 335         0x9180, 0xffffffff, 0x00050004,
 336         0x918c, 0xffffffff, 0x00010006,
 337         0x9190, 0xffffffff, 0x00090008,
 338         0x9194, 0xffffffff, 0x00070000,
 339         0x9198, 0xffffffff, 0x00030002,
 340         0x919c, 0xffffffff, 0x00050004,
 341         0x91a8, 0xffffffff, 0x00010006,
 342         0x91ac, 0xffffffff, 0x00090008,
 343         0x91b0, 0xffffffff, 0x00070000,
 344         0x91b4, 0xffffffff, 0x00030002,
 345         0x91b8, 0xffffffff, 0x00050004,
 346         0x91c4, 0xffffffff, 0x00010006,
 347         0x91c8, 0xffffffff, 0x00090008,
 348         0x91cc, 0xffffffff, 0x00070000,
 349         0x91d0, 0xffffffff, 0x00030002,
 350         0x91d4, 0xffffffff, 0x00050004,
 351         0x91e0, 0xffffffff, 0x00010006,
 352         0x91e4, 0xffffffff, 0x00090008,
 353         0x91e8, 0xffffffff, 0x00000000,
 354         0x91ec, 0xffffffff, 0x00070000,
 355         0x91f0, 0xffffffff, 0x00030002,
 356         0x91f4, 0xffffffff, 0x00050004,
 357         0x9200, 0xffffffff, 0x00010006,
 358         0x9204, 0xffffffff, 0x00090008,
 359         0x9208, 0xffffffff, 0x00070000,
 360         0x920c, 0xffffffff, 0x00030002,
 361         0x9210, 0xffffffff, 0x00050004,
 362         0x921c, 0xffffffff, 0x00010006,
 363         0x9220, 0xffffffff, 0x00090008,
 364         0x9224, 0xffffffff, 0x00070000,
 365         0x9228, 0xffffffff, 0x00030002,
 366         0x922c, 0xffffffff, 0x00050004,
 367         0x9238, 0xffffffff, 0x00010006,
 368         0x923c, 0xffffffff, 0x00090008,
 369         0x9240, 0xffffffff, 0x00070000,
 370         0x9244, 0xffffffff, 0x00030002,
 371         0x9248, 0xffffffff, 0x00050004,
 372         0x9254, 0xffffffff, 0x00010006,
 373         0x9258, 0xffffffff, 0x00090008,
 374         0x925c, 0xffffffff, 0x00070000,
 375         0x9260, 0xffffffff, 0x00030002,
 376         0x9264, 0xffffffff, 0x00050004,
 377         0x9270, 0xffffffff, 0x00010006,
 378         0x9274, 0xffffffff, 0x00090008,
 379         0x9278, 0xffffffff, 0x00070000,
 380         0x927c, 0xffffffff, 0x00030002,
 381         0x9280, 0xffffffff, 0x00050004,
 382         0x928c, 0xffffffff, 0x00010006,
 383         0x9290, 0xffffffff, 0x00090008,
 384         0x9294, 0xffffffff, 0x00000000,
 385         0x929c, 0xffffffff, 0x00000001,
 386         0x802c, 0xffffffff, 0x40010000,
 387         0x915c, 0xffffffff, 0x00010000,
 388         0x9160, 0xffffffff, 0x00030002,
 389         0x9178, 0xffffffff, 0x00070000,
 390         0x917c, 0xffffffff, 0x00030002,
 391         0x9180, 0xffffffff, 0x00050004,
 392         0x918c, 0xffffffff, 0x00010006,
 393         0x9190, 0xffffffff, 0x00090008,
 394         0x9194, 0xffffffff, 0x00070000,
 395         0x9198, 0xffffffff, 0x00030002,
 396         0x919c, 0xffffffff, 0x00050004,
 397         0x91a8, 0xffffffff, 0x00010006,
 398         0x91ac, 0xffffffff, 0x00090008,
 399         0x91b0, 0xffffffff, 0x00070000,
 400         0x91b4, 0xffffffff, 0x00030002,
 401         0x91b8, 0xffffffff, 0x00050004,
 402         0x91c4, 0xffffffff, 0x00010006,
 403         0x91c8, 0xffffffff, 0x00090008,
 404         0x91cc, 0xffffffff, 0x00070000,
 405         0x91d0, 0xffffffff, 0x00030002,
 406         0x91d4, 0xffffffff, 0x00050004,
 407         0x91e0, 0xffffffff, 0x00010006,
 408         0x91e4, 0xffffffff, 0x00090008,
 409         0x91e8, 0xffffffff, 0x00000000,
 410         0x91ec, 0xffffffff, 0x00070000,
 411         0x91f0, 0xffffffff, 0x00030002,
 412         0x91f4, 0xffffffff, 0x00050004,
 413         0x9200, 0xffffffff, 0x00010006,
 414         0x9204, 0xffffffff, 0x00090008,
 415         0x9208, 0xffffffff, 0x00070000,
 416         0x920c, 0xffffffff, 0x00030002,
 417         0x9210, 0xffffffff, 0x00050004,
 418         0x921c, 0xffffffff, 0x00010006,
 419         0x9220, 0xffffffff, 0x00090008,
 420         0x9224, 0xffffffff, 0x00070000,
 421         0x9228, 0xffffffff, 0x00030002,
 422         0x922c, 0xffffffff, 0x00050004,
 423         0x9238, 0xffffffff, 0x00010006,
 424         0x923c, 0xffffffff, 0x00090008,
 425         0x9240, 0xffffffff, 0x00070000,
 426         0x9244, 0xffffffff, 0x00030002,
 427         0x9248, 0xffffffff, 0x00050004,
 428         0x9254, 0xffffffff, 0x00010006,
 429         0x9258, 0xffffffff, 0x00090008,
 430         0x925c, 0xffffffff, 0x00070000,
 431         0x9260, 0xffffffff, 0x00030002,
 432         0x9264, 0xffffffff, 0x00050004,
 433         0x9270, 0xffffffff, 0x00010006,
 434         0x9274, 0xffffffff, 0x00090008,
 435         0x9278, 0xffffffff, 0x00070000,
 436         0x927c, 0xffffffff, 0x00030002,
 437         0x9280, 0xffffffff, 0x00050004,
 438         0x928c, 0xffffffff, 0x00010006,
 439         0x9290, 0xffffffff, 0x00090008,
 440         0x9294, 0xffffffff, 0x00000000,
 441         0x929c, 0xffffffff, 0x00000001,
 442         0x802c, 0xffffffff, 0xc0000000
 443 };
 444
 445 static const u32 redwood_mgcg_init[] =
 446 {
 447         0x802c, 0xffffffff, 0xc0000000,
 448         0x5448, 0xffffffff, 0x00000100,
 449         0x55e4, 0xffffffff, 0x00000100,
 450         0x160c, 0xffffffff, 0x00000100,
 451         0x5644, 0xffffffff, 0x00000100,
 452         0xc164, 0xffffffff, 0x00000100,
 453         0x8a18, 0xffffffff, 0x00000100,
 454         0x897c, 0xffffffff, 0x06000100,
 455         0x8b28, 0xffffffff, 0x00000100,
 456         0x9144, 0xffffffff, 0x00000100,
 457         0x9a60, 0xffffffff, 0x00000100,
 458         0x9868, 0xffffffff, 0x00000100,
 459         0x8d58, 0xffffffff, 0x00000100,
 460         0x9510, 0xffffffff, 0x00000100,
 461         0x949c, 0xffffffff, 0x00000100,
 462         0x9654, 0xffffffff, 0x00000100,
 463         0x9030, 0xffffffff, 0x00000100,
 464         0x9034, 0xffffffff, 0x00000100,
 465         0x9038, 0xffffffff, 0x00000100,
 466         0x903c, 0xffffffff, 0x00000100,
 467         0x9040, 0xffffffff, 0x00000100,
 468         0xa200, 0xffffffff, 0x00000100,
 469         0xa204, 0xffffffff, 0x00000100,
 470         0xa208, 0xffffffff, 0x00000100,
 471         0xa20c, 0xffffffff, 0x00000100,
 472         0x971c, 0xffffffff, 0x00000100,
 473         0x977c, 0xffffffff, 0x00000100,
 474         0x3f80, 0xffffffff, 0x00000100,
 475         0xa210, 0xffffffff, 0x00000100,
 476         0xa214, 0xffffffff, 0x00000100,
 477         0x4d8, 0xffffffff, 0x00000100,
 478         0x9784, 0xffffffff, 0x00000100,
 479         0x9698, 0xffffffff, 0x00000100,
 480         0x4d4, 0xffffffff, 0x00000200,
 481         0x30cc, 0xffffffff, 0x00000100,
 482         0xd0c0, 0xffffffff, 0xff000100,
 483         0x802c, 0xffffffff, 0x40000000,
 484         0x915c, 0xffffffff, 0x00010000,
 485         0x9160, 0xffffffff, 0x00030002,
 486         0x9178, 0xffffffff, 0x00070000,
 487         0x917c, 0xffffffff, 0x00030002,
 488         0x9180, 0xffffffff, 0x00050004,
 489         0x918c, 0xffffffff, 0x00010006,
 490         0x9190, 0xffffffff, 0x00090008,
 491         0x9194, 0xffffffff, 0x00070000,
 492         0x9198, 0xffffffff, 0x00030002,
 493         0x919c, 0xffffffff, 0x00050004,
 494         0x91a8, 0xffffffff, 0x00010006,
 495         0x91ac, 0xffffffff, 0x00090008,
 496         0x91b0, 0xffffffff, 0x00070000,
 497         0x91b4, 0xffffffff, 0x00030002,
 498         0x91b8, 0xffffffff, 0x00050004,
 499         0x91c4, 0xffffffff, 0x00010006,
 500         0x91c8, 0xffffffff, 0x00090008,
 501         0x91cc, 0xffffffff, 0x00070000,
 502         0x91d0, 0xffffffff, 0x00030002,
 503         0x91d4, 0xffffffff, 0x00050004,
 504         0x91e0, 0xffffffff, 0x00010006,
 505         0x91e4, 0xffffffff, 0x00090008,
 506         0x91e8, 0xffffffff, 0x00000000,
 507         0x91ec, 0xffffffff, 0x00070000,
 508         0x91f0, 0xffffffff, 0x00030002,
 509         0x91f4, 0xffffffff, 0x00050004,
 510         0x9200, 0xffffffff, 0x00010006,
 511         0x9204, 0xffffffff, 0x00090008,
 512         0x9294, 0xffffffff, 0x00000000,
 513         0x929c, 0xffffffff, 0x00000001,
 514         0x802c, 0xffffffff, 0xc0000000
 515 };
 516
 517 static const u32 cedar_golden_registers[] =
 518 {
 519         0x3f90, 0xffff0000, 0xff000000,
 520         0x9148, 0xffff0000, 0xff000000,
 521         0x3f94, 0xffff0000, 0xff000000,
 522         0x914c, 0xffff0000, 0xff000000,
 523         0x9b7c, 0xffffffff, 0x00000000,
 524         0x8a14, 0xffffffff, 0x00000007,
 525         0x8b10, 0xffffffff, 0x00000000,
 526         0x960c, 0xffffffff, 0x54763210,
 527         0x88c4, 0xffffffff, 0x000000c2,
 528         0x88d4, 0xffffffff, 0x00000000,
 529         0x8974, 0xffffffff, 0x00000000,
 530         0xc78, 0x00000080, 0x00000080,
 531         0x5eb4, 0xffffffff, 0x00000002,
 532         0x5e78, 0xffffffff, 0x001000f0,
 533         0x6104, 0x01000300, 0x00000000,
 534         0x5bc0, 0x00300000, 0x00000000,
 535         0x7030, 0xffffffff, 0x00000011,
 536         0x7c30, 0xffffffff, 0x00000011,
 537         0x10830, 0xffffffff, 0x00000011,
 538         0x11430, 0xffffffff, 0x00000011,
 539         0xd02c, 0xffffffff, 0x08421000,
 540         0x240c, 0xffffffff, 0x00000380,
 541         0x8b24, 0xffffffff, 0x00ff0fff,
 542         0x28a4c, 0x06000000, 0x06000000,
 543         0x10c, 0x00000001, 0x00000001,
 544         0x8d00, 0xffffffff, 0x100e4848,
 545         0x8d04, 0xffffffff, 0x00164745,
 546         0x8c00, 0xffffffff, 0xe4000003,
 547         0x8c04, 0xffffffff, 0x40600060,
 548         0x8c08, 0xffffffff, 0x001c001c,
 549         0x8cf0, 0xffffffff, 0x08e00410,
 550         0x8c20, 0xffffffff, 0x00800080,
 551         0x8c24, 0xffffffff, 0x00800080,
 552         0x8c18, 0xffffffff, 0x20202078,
 553         0x8c1c, 0xffffffff, 0x00001010,
 554         0x28350, 0xffffffff, 0x00000000,
 555         0xa008, 0xffffffff, 0x00010000,
 556         0x5c4, 0xffffffff, 0x00000001,
 557         0x9508, 0xffffffff, 0x00000002
 558 };
 559
 560 static const u32 cedar_mgcg_init[] =
 561 {
 562         0x802c, 0xffffffff, 0xc0000000,
 563         0x5448, 0xffffffff, 0x00000100,
 564         0x55e4, 0xffffffff, 0x00000100,
 565         0x160c, 0xffffffff, 0x00000100,
 566         0x5644, 0xffffffff, 0x00000100,
 567         0xc164, 0xffffffff, 0x00000100,
 568         0x8a18, 0xffffffff, 0x00000100,
 569         0x897c, 0xffffffff, 0x06000100,
 570         0x8b28, 0xffffffff, 0x00000100,
 571         0x9144, 0xffffffff, 0x00000100,
 572         0x9a60, 0xffffffff, 0x00000100,
 573         0x9868, 0xffffffff, 0x00000100,
 574         0x8d58, 0xffffffff, 0x00000100,
 575         0x9510, 0xffffffff, 0x00000100,
 576         0x949c, 0xffffffff, 0x00000100,
 577         0x9654, 0xffffffff, 0x00000100,
 578         0x9030, 0xffffffff, 0x00000100,
 579         0x9034, 0xffffffff, 0x00000100,
 580         0x9038, 0xffffffff, 0x00000100,
 581         0x903c, 0xffffffff, 0x00000100,
 582         0x9040, 0xffffffff, 0x00000100,
 583         0xa200, 0xffffffff, 0x00000100,
 584         0xa204, 0xffffffff, 0x00000100,
 585         0xa208, 0xffffffff, 0x00000100,
 586         0xa20c, 0xffffffff, 0x00000100,
 587         0x971c, 0xffffffff, 0x00000100,
 588         0x977c, 0xffffffff, 0x00000100,
 589         0x3f80, 0xffffffff, 0x00000100,
 590         0xa210, 0xffffffff, 0x00000100,
 591         0xa214, 0xffffffff, 0x00000100,
 592         0x4d8, 0xffffffff, 0x00000100,
 593         0x9784, 0xffffffff, 0x00000100,
 594         0x9698, 0xffffffff, 0x00000100,
 595         0x4d4, 0xffffffff, 0x00000200,
 596         0x30cc, 0xffffffff, 0x00000100,
 597         0xd0c0, 0xffffffff, 0xff000100,
 598         0x802c, 0xffffffff, 0x40000000,
 599         0x915c, 0xffffffff, 0x00010000,
 600         0x9178, 0xffffffff, 0x00050000,
 601         0x917c, 0xffffffff, 0x00030002,
 602         0x918c, 0xffffffff, 0x00010004,
 603         0x9190, 0xffffffff, 0x00070006,
 604         0x9194, 0xffffffff, 0x00050000,
 605         0x9198, 0xffffffff, 0x00030002,
 606         0x91a8, 0xffffffff, 0x00010004,
 607         0x91ac, 0xffffffff, 0x00070006,
 608         0x91e8, 0xffffffff, 0x00000000,
 609         0x9294, 0xffffffff, 0x00000000,
 610         0x929c, 0xffffffff, 0x00000001,
 611         0x802c, 0xffffffff, 0xc0000000
 612 };
 613
 614 static const u32 juniper_mgcg_init[] =
 615 {
 616         0x802c, 0xffffffff, 0xc0000000,
 617         0x5448, 0xffffffff, 0x00000100,
 618         0x55e4, 0xffffffff, 0x00000100,
 619         0x160c, 0xffffffff, 0x00000100,
 620         0x5644, 0xffffffff, 0x00000100,
 621         0xc164, 0xffffffff, 0x00000100,
 622         0x8a18, 0xffffffff, 0x00000100,
 623         0x897c, 0xffffffff, 0x06000100,
 624         0x8b28, 0xffffffff, 0x00000100,
 625         0x9144, 0xffffffff, 0x00000100,
 626         0x9a60, 0xffffffff, 0x00000100,
 627         0x9868, 0xffffffff, 0x00000100,
 628         0x8d58, 0xffffffff, 0x00000100,
 629         0x9510, 0xffffffff, 0x00000100,
 630         0x949c, 0xffffffff, 0x00000100,
 631         0x9654, 0xffffffff, 0x00000100,
 632         0x9030, 0xffffffff, 0x00000100,
 633         0x9034, 0xffffffff, 0x00000100,
 634         0x9038, 0xffffffff, 0x00000100,
 635         0x903c, 0xffffffff, 0x00000100,
 636         0x9040, 0xffffffff, 0x00000100,
 637         0xa200, 0xffffffff, 0x00000100,
 638         0xa204, 0xffffffff, 0x00000100,
 639         0xa208, 0xffffffff, 0x00000100,
 640         0xa20c, 0xffffffff, 0x00000100,
 641         0x971c, 0xffffffff, 0x00000100,
 642         0xd0c0, 0xffffffff, 0xff000100,
 643         0x802c, 0xffffffff, 0x40000000,
 644         0x915c, 0xffffffff, 0x00010000,
 645         0x9160, 0xffffffff, 0x00030002,
 646         0x9178, 0xffffffff, 0x00070000,
 647         0x917c, 0xffffffff, 0x00030002,
 648         0x9180, 0xffffffff, 0x00050004,
 649         0x918c, 0xffffffff, 0x00010006,
 650         0x9190, 0xffffffff, 0x00090008,
 651         0x9194, 0xffffffff, 0x00070000,
 652         0x9198, 0xffffffff, 0x00030002,
 653         0x919c, 0xffffffff, 0x00050004,
 654         0x91a8, 0xffffffff, 0x00010006,
 655         0x91ac, 0xffffffff, 0x00090008,
 656         0x91b0, 0xffffffff, 0x00070000,
 657         0x91b4, 0xffffffff, 0x00030002,
 658         0x91b8, 0xffffffff, 0x00050004,
 659         0x91c4, 0xffffffff, 0x00010006,
 660         0x91c8, 0xffffffff, 0x00090008,
 661         0x91cc, 0xffffffff, 0x00070000,
 662         0x91d0, 0xffffffff, 0x00030002,
 663         0x91d4, 0xffffffff, 0x00050004,
 664         0x91e0, 0xffffffff, 0x00010006,
 665         0x91e4, 0xffffffff, 0x00090008,
 666         0x91e8, 0xffffffff, 0x00000000,
 667         0x91ec, 0xffffffff, 0x00070000,
 668         0x91f0, 0xffffffff, 0x00030002,
 669         0x91f4, 0xffffffff, 0x00050004,
 670         0x9200, 0xffffffff, 0x00010006,
 671         0x9204, 0xffffffff, 0x00090008,
 672         0x9208, 0xffffffff, 0x00070000,
 673         0x920c, 0xffffffff, 0x00030002,
 674         0x9210, 0xffffffff, 0x00050004,
 675         0x921c, 0xffffffff, 0x00010006,
 676         0x9220, 0xffffffff, 0x00090008,
 677         0x9224, 0xffffffff, 0x00070000,
 678         0x9228, 0xffffffff, 0x00030002,
 679         0x922c, 0xffffffff, 0x00050004,
 680         0x9238, 0xffffffff, 0x00010006,
 681         0x923c, 0xffffffff, 0x00090008,
 682         0x9240, 0xffffffff, 0x00070000,
 683         0x9244, 0xffffffff, 0x00030002,
 684         0x9248, 0xffffffff, 0x00050004,
 685         0x9254, 0xffffffff, 0x00010006,
 686         0x9258, 0xffffffff, 0x00090008,
 687         0x925c, 0xffffffff, 0x00070000,
 688         0x9260, 0xffffffff, 0x00030002,
 689         0x9264, 0xffffffff, 0x00050004,
 690         0x9270, 0xffffffff, 0x00010006,
 691         0x9274, 0xffffffff, 0x00090008,
 692         0x9278, 0xffffffff, 0x00070000,
 693         0x927c, 0xffffffff, 0x00030002,
 694         0x9280, 0xffffffff, 0x00050004,
 695         0x928c, 0xffffffff, 0x00010006,
 696         0x9290, 0xffffffff, 0x00090008,
 697         0x9294, 0xffffffff, 0x00000000,
 698         0x929c, 0xffffffff, 0x00000001,
 699         0x802c, 0xffffffff, 0xc0000000,
 700         0x977c, 0xffffffff, 0x00000100,
 701         0x3f80, 0xffffffff, 0x00000100,
 702         0xa210, 0xffffffff, 0x00000100,
 703         0xa214, 0xffffffff, 0x00000100,
 704         0x4d8, 0xffffffff, 0x00000100,
 705         0x9784, 0xffffffff, 0x00000100,
 706         0x9698, 0xffffffff, 0x00000100,
 707         0x4d4, 0xffffffff, 0x00000200,
 708         0x30cc, 0xffffffff, 0x00000100,
 709         0x802c, 0xffffffff, 0xc0000000
 710 };
 711
 712 static const u32 supersumo_golden_registers[] =
 713 {
 714         0x5eb4, 0xffffffff, 0x00000002,
 715         0x5c4, 0xffffffff, 0x00000001,
 716         0x7030, 0xffffffff, 0x00000011,
 717         0x7c30, 0xffffffff, 0x00000011,
 718         0x6104, 0x01000300, 0x00000000,
 719         0x5bc0, 0x00300000, 0x00000000,
 720         0x8c04, 0xffffffff, 0x40600060,
 721         0x8c08, 0xffffffff, 0x001c001c,
 722         0x8c20, 0xffffffff, 0x00800080,
 723         0x8c24, 0xffffffff, 0x00800080,
 724         0x8c18, 0xffffffff, 0x20202078,
 725         0x8c1c, 0xffffffff, 0x00001010,
 726         0x918c, 0xffffffff, 0x00010006,
 727         0x91a8, 0xffffffff, 0x00010006,
 728         0x91c4, 0xffffffff, 0x00010006,
 729         0x91e0, 0xffffffff, 0x00010006,
 730         0x9200, 0xffffffff, 0x00010006,
 731         0x9150, 0xffffffff, 0x6e944040,
 732         0x917c, 0xffffffff, 0x00030002,
 733         0x9180, 0xffffffff, 0x00050004,
 734         0x9198, 0xffffffff, 0x00030002,
 735         0x919c, 0xffffffff, 0x00050004,
 736         0x91b4, 0xffffffff, 0x00030002,
 737         0x91b8, 0xffffffff, 0x00050004,
 738         0x91d0, 0xffffffff, 0x00030002,
 739         0x91d4, 0xffffffff, 0x00050004,
 740         0x91f0, 0xffffffff, 0x00030002,
 741         0x91f4, 0xffffffff, 0x00050004,
 742         0x915c, 0xffffffff, 0x00010000,
 743         0x9160, 0xffffffff, 0x00030002,
 744         0x3f90, 0xffff0000, 0xff000000,
 745         0x9178, 0xffffffff, 0x00070000,
 746         0x9194, 0xffffffff, 0x00070000,
 747         0x91b0, 0xffffffff, 0x00070000,
 748         0x91cc, 0xffffffff, 0x00070000,
 749         0x91ec, 0xffffffff, 0x00070000,
 750         0x9148, 0xffff0000, 0xff000000,
 751         0x9190, 0xffffffff, 0x00090008,
 752         0x91ac, 0xffffffff, 0x00090008,
 753         0x91c8, 0xffffffff, 0x00090008,
 754         0x91e4, 0xffffffff, 0x00090008,
 755         0x9204, 0xffffffff, 0x00090008,
 756         0x3f94, 0xffff0000, 0xff000000,
 757         0x914c, 0xffff0000, 0xff000000,
 758         0x929c, 0xffffffff, 0x00000001,
 759         0x8a18, 0xffffffff, 0x00000100,
 760         0x8b28, 0xffffffff, 0x00000100,
 761         0x9144, 0xffffffff, 0x00000100,
 762         0x5644, 0xffffffff, 0x00000100,
 763         0x9b7c, 0xffffffff, 0x00000000,
 764         0x8030, 0xffffffff, 0x0000100a,
 765         0x8a14, 0xffffffff, 0x00000007,
 766         0x8b24, 0xffffffff, 0x00ff0fff,
 767         0x8b10, 0xffffffff, 0x00000000,
 768         0x28a4c, 0x06000000, 0x06000000,
 769         0x4d8, 0xffffffff, 0x00000100,
 770         0x913c, 0xffff000f, 0x0100000a,
 771         0x960c, 0xffffffff, 0x54763210,
 772         0x88c4, 0xffffffff, 0x000000c2,
 773         0x88d4, 0xffffffff, 0x00000010,
 774         0x8974, 0xffffffff, 0x00000000,
 775         0xc78, 0x00000080, 0x00000080,
 776         0x5e78, 0xffffffff, 0x001000f0,
 777         0xd02c, 0xffffffff, 0x08421000,
 778         0xa008, 0xffffffff, 0x00010000,
 779         0x8d00, 0xffffffff, 0x100e4848,
 780         0x8d04, 0xffffffff, 0x00164745,
 781         0x8c00, 0xffffffff, 0xe4000003,
 782         0x8cf0, 0x1fffffff, 0x08e00620,
 783         0x28350, 0xffffffff, 0x00000000,
 784         0x9508, 0xffffffff, 0x00000002
 785 };
 786
 787 static const u32 sumo_golden_registers[] =
 788 {
 789         0x900c, 0x00ffffff, 0x0017071f,
 790         0x8c18, 0xffffffff, 0x10101060,
 791         0x8c1c, 0xffffffff, 0x00001010,
 792         0x8c30, 0x0000000f, 0x00000005,
 793         0x9688, 0x0000000f, 0x00000007
 794 };
 795
 796 static const u32 wrestler_golden_registers[] =
 797 {
 798         0x5eb4, 0xffffffff, 0x00000002,
 799         0x5c4, 0xffffffff, 0x00000001,
 800         0x7030, 0xffffffff, 0x00000011,
 801         0x7c30, 0xffffffff, 0x00000011,
 802         0x6104, 0x01000300, 0x00000000,
 803         0x5bc0, 0x00300000, 0x00000000,
 804         0x918c, 0xffffffff, 0x00010006,
 805         0x91a8, 0xffffffff, 0x00010006,
 806         0x9150, 0xffffffff, 0x6e944040,
 807         0x917c, 0xffffffff, 0x00030002,
 808         0x9198, 0xffffffff, 0x00030002,
 809         0x915c, 0xffffffff, 0x00010000,
 810         0x3f90, 0xffff0000, 0xff000000,
 811         0x9178, 0xffffffff, 0x00070000,
 812         0x9194, 0xffffffff, 0x00070000,
 813         0x9148, 0xffff0000, 0xff000000,
 814         0x9190, 0xffffffff, 0x00090008,
 815         0x91ac, 0xffffffff, 0x00090008,
 816         0x3f94, 0xffff0000, 0xff000000,
 817         0x914c, 0xffff0000, 0xff000000,
 818         0x929c, 0xffffffff, 0x00000001,
 819         0x8a18, 0xffffffff, 0x00000100,
 820         0x8b28, 0xffffffff, 0x00000100,
 821         0x9144, 0xffffffff, 0x00000100,
 822         0x9b7c, 0xffffffff, 0x00000000,
 823         0x8030, 0xffffffff, 0x0000100a,
 824         0x8a14, 0xffffffff, 0x00000001,
 825         0x8b24, 0xffffffff, 0x00ff0fff,
 826         0x8b10, 0xffffffff, 0x00000000,
 827         0x28a4c, 0x06000000, 0x06000000,
 828         0x4d8, 0xffffffff, 0x00000100,
 829         0x913c, 0xffff000f, 0x0100000a,
 830         0x960c, 0xffffffff, 0x54763210,
 831         0x88c4, 0xffffffff, 0x000000c2,
 832         0x88d4, 0xffffffff, 0x00000010,
 833         0x8974, 0xffffffff, 0x00000000,
 834         0xc78, 0x00000080, 0x00000080,
 835         0x5e78, 0xffffffff, 0x001000f0,
 836         0xd02c, 0xffffffff, 0x08421000,
 837         0xa008, 0xffffffff, 0x00010000,
 838         0x8d00, 0xffffffff, 0x100e4848,
 839         0x8d04, 0xffffffff, 0x00164745,
 840         0x8c00, 0xffffffff, 0xe4000003,
 841         0x8cf0, 0x1fffffff, 0x08e00410,
 842         0x28350, 0xffffffff, 0x00000000,
 843         0x9508, 0xffffffff, 0x00000002,
 844         0x900c, 0xffffffff, 0x0017071f,
 845         0x8c18, 0xffffffff, 0x10101060,
 846         0x8c1c, 0xffffffff, 0x00001010
 847 };
 848
 849 static const u32 barts_golden_registers[] =
 850 {
 851         0x5eb4, 0xffffffff, 0x00000002,
 852         0x5e78, 0x8f311ff1, 0x001000f0,
 853         0x3f90, 0xffff0000, 0xff000000,
 854         0x9148, 0xffff0000, 0xff000000,
 855         0x3f94, 0xffff0000, 0xff000000,
 856         0x914c, 0xffff0000, 0xff000000,
 857         0xc78, 0x00000080, 0x00000080,
 858         0xbd4, 0x70073777, 0x00010001,
 859         0xd02c, 0xbfffff1f, 0x08421000,
 860         0xd0b8, 0x03773777, 0x02011003,
 861         0x5bc0, 0x00200000, 0x50100000,
 862         0x98f8, 0x33773777, 0x02011003,
 863         0x98fc, 0xffffffff, 0x76543210,
 864         0x7030, 0x31000311, 0x00000011,
 865         0x2f48, 0x00000007, 0x02011003,
 866         0x6b28, 0x00000010, 0x00000012,
 867         0x7728, 0x00000010, 0x00000012,
 868         0x10328, 0x00000010, 0x00000012,
 869         0x10f28, 0x00000010, 0x00000012,
 870         0x11b28, 0x00000010, 0x00000012,
 871         0x12728, 0x00000010, 0x00000012,
 872         0x240c, 0x000007ff, 0x00000380,
 873         0x8a14, 0xf000001f, 0x00000007,
 874         0x8b24, 0x3fff3fff, 0x00ff0fff,
 875         0x8b10, 0x0000ff0f, 0x00000000,
 876         0x28a4c, 0x07ffffff, 0x06000000,
 877         0x10c, 0x00000001, 0x00010003,
 878         0xa02c, 0xffffffff, 0x0000009b,
 879         0x913c, 0x0000000f, 0x0100000a,
 880         0x8d00, 0xffff7f7f, 0x100e4848,
 881         0x8d04, 0x00ffffff, 0x00164745,
 882         0x8c00, 0xfffc0003, 0xe4000003,
 883         0x8c04, 0xf8ff00ff, 0x40600060,
 884         0x8c08, 0x00ff00ff, 0x001c001c,
 885         0x8cf0, 0x1fff1fff, 0x08e00620,
 886         0x8c20, 0x0fff0fff, 0x00800080,
 887         0x8c24, 0x0fff0fff, 0x00800080,
 888         0x8c18, 0xffffffff, 0x20202078,
 889         0x8c1c, 0x0000ffff, 0x00001010,
 890         0x28350, 0x00000f01, 0x00000000,
 891         0x9508, 0x3700001f, 0x00000002,
 892         0x960c, 0xffffffff, 0x54763210,
 893         0x88c4, 0x001f3ae3, 0x000000c2,
 894         0x88d4, 0x0000001f, 0x00000010,
 895         0x8974, 0xffffffff, 0x00000000
 896 };
 897
 898 static const u32 turks_golden_registers[] =
 899 {
 900         0x5eb4, 0xffffffff, 0x00000002,
 901         0x5e78, 0x8f311ff1, 0x001000f0,
 902         0x8c8, 0x00003000, 0x00001070,
 903         0x8cc, 0x000fffff, 0x00040035,
 904         0x3f90, 0xffff0000, 0xfff00000,
 905         0x9148, 0xffff0000, 0xfff00000,
 906         0x3f94, 0xffff0000, 0xfff00000,
 907         0x914c, 0xffff0000, 0xfff00000,
 908         0xc78, 0x00000080, 0x00000080,
 909         0xbd4, 0x00073007, 0x00010002,
 910         0xd02c, 0xbfffff1f, 0x08421000,
 911         0xd0b8, 0x03773777, 0x02010002,
 912         0x5bc0, 0x00200000, 0x50100000,
 913         0x98f8, 0x33773777, 0x00010002,
 914         0x98fc, 0xffffffff, 0x33221100,
 915         0x7030, 0x31000311, 0x00000011,
 916         0x2f48, 0x33773777, 0x00010002,
 917         0x6b28, 0x00000010, 0x00000012,
 918         0x7728, 0x00000010, 0x00000012,
 919         0x10328, 0x00000010, 0x00000012,
 920         0x10f28, 0x00000010, 0x00000012,
 921         0x11b28, 0x00000010, 0x00000012,
 922         0x12728, 0x00000010, 0x00000012,
 923         0x240c, 0x000007ff, 0x00000380,
 924         0x8a14, 0xf000001f, 0x00000007,
 925         0x8b24, 0x3fff3fff, 0x00ff0fff,
 926         0x8b10, 0x0000ff0f, 0x00000000,
 927         0x28a4c, 0x07ffffff, 0x06000000,
 928         0x10c, 0x00000001, 0x00010003,
 929         0xa02c, 0xffffffff, 0x0000009b,
 930         0x913c, 0x0000000f, 0x0100000a,
 931         0x8d00, 0xffff7f7f, 0x100e4848,
 932         0x8d04, 0x00ffffff, 0x00164745,
 933         0x8c00, 0xfffc0003, 0xe4000003,
 934         0x8c04, 0xf8ff00ff, 0x40600060,
 935         0x8c08, 0x00ff00ff, 0x001c001c,
 936         0x8cf0, 0x1fff1fff, 0x08e00410,
 937         0x8c20, 0x0fff0fff, 0x00800080,
 938         0x8c24, 0x0fff0fff, 0x00800080,
 939         0x8c18, 0xffffffff, 0x20202078,
 940         0x8c1c, 0x0000ffff, 0x00001010,
 941         0x28350, 0x00000f01, 0x00000000,
 942         0x9508, 0x3700001f, 0x00000002,
 943         0x960c, 0xffffffff, 0x54763210,
 944         0x88c4, 0x001f3ae3, 0x000000c2,
 945         0x88d4, 0x0000001f, 0x00000010,
 946         0x8974, 0xffffffff, 0x00000000
 947 };
 948
 949 static const u32 caicos_golden_registers[] =
 950 {
 951         0x5eb4, 0xffffffff, 0x00000002,
 952         0x5e78, 0x8f311ff1, 0x001000f0,
 953         0x8c8, 0x00003420, 0x00001450,
 954         0x8cc, 0x000fffff, 0x00040035,
 955         0x3f90, 0xffff0000, 0xfffc0000,
 956         0x9148, 0xffff0000, 0xfffc0000,
 957         0x3f94, 0xffff0000, 0xfffc0000,
 958         0x914c, 0xffff0000, 0xfffc0000,
 959         0xc78, 0x00000080, 0x00000080,
 960         0xbd4, 0x00073007, 0x00010001,
 961         0xd02c, 0xbfffff1f, 0x08421000,
 962         0xd0b8, 0x03773777, 0x02010001,
 963         0x5bc0, 0x00200000, 0x50100000,
 964         0x98f8, 0x33773777, 0x02010001,
 965         0x98fc, 0xffffffff, 0x33221100,
 966         0x7030, 0x31000311, 0x00000011,
 967         0x2f48, 0x33773777, 0x02010001,
 968         0x6b28, 0x00000010, 0x00000012,
 969         0x7728, 0x00000010, 0x00000012,
 970         0x10328, 0x00000010, 0x00000012,
 971         0x10f28, 0x00000010, 0x00000012,
 972         0x11b28, 0x00000010, 0x00000012,
 973         0x12728, 0x00000010, 0x00000012,
 974         0x240c, 0x000007ff, 0x00000380,
 975         0x8a14, 0xf000001f, 0x00000001,
 976         0x8b24, 0x3fff3fff, 0x00ff0fff,
 977         0x8b10, 0x0000ff0f, 0x00000000,
 978         0x28a4c, 0x07ffffff, 0x06000000,
 979         0x10c, 0x00000001, 0x00010003,
 980         0xa02c, 0xffffffff, 0x0000009b,
 981         0x913c, 0x0000000f, 0x0100000a,
 982         0x8d00, 0xffff7f7f, 0x100e4848,
 983         0x8d04, 0x00ffffff, 0x00164745,
 984         0x8c00, 0xfffc0003, 0xe4000003,
 985         0x8c04, 0xf8ff00ff, 0x40600060,
 986         0x8c08, 0x00ff00ff, 0x001c001c,
 987         0x8cf0, 0x1fff1fff, 0x08e00410,
 988         0x8c20, 0x0fff0fff, 0x00800080,
 989         0x8c24, 0x0fff0fff, 0x00800080,
 990         0x8c18, 0xffffffff, 0x20202078,
 991         0x8c1c, 0x0000ffff, 0x00001010,
 992         0x28350, 0x00000f01, 0x00000000,
 993         0x9508, 0x3700001f, 0x00000002,
 994         0x960c, 0xffffffff, 0x54763210,
 995         0x88c4, 0x001f3ae3, 0x000000c2,
 996         0x88d4, 0x0000001f, 0x00000010,
 997         0x8974, 0xffffffff, 0x00000000
 998 };
 999
1000 static void evergreen_init_golden_registers(struct radeon_device *rdev)
1001 {
1002         switch (rdev->family) {
1003         case CHIP_CYPRESS:
1004         case CHIP_HEMLOCK:
1005                 radeon_program_register_sequence(rdev,
1006                                                  evergreen_golden_registers,
1007                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1008                 radeon_program_register_sequence(rdev,
1009                                                  evergreen_golden_registers2,
1010                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1011                 radeon_program_register_sequence(rdev,
1012                                                  cypress_mgcg_init,
1013                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
1014                 break;
1015         case CHIP_JUNIPER:
1016                 radeon_program_register_sequence(rdev,
1017                                                  evergreen_golden_registers,
1018                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1019                 radeon_program_register_sequence(rdev,
1020                                                  evergreen_golden_registers2,
1021                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1022                 radeon_program_register_sequence(rdev,
1023                                                  juniper_mgcg_init,
1024                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
1025                 break;
1026         case CHIP_REDWOOD:
1027                 radeon_program_register_sequence(rdev,
1028                                                  evergreen_golden_registers,
1029                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1030                 radeon_program_register_sequence(rdev,
1031                                                  evergreen_golden_registers2,
1032                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1033                 radeon_program_register_sequence(rdev,
1034                                                  redwood_mgcg_init,
1035                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
1036                 break;
1037         case CHIP_CEDAR:
1038                 radeon_program_register_sequence(rdev,
1039                                                  cedar_golden_registers,
1040                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
1041                 radeon_program_register_sequence(rdev,
1042                                                  evergreen_golden_registers2,
1043                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1044                 radeon_program_register_sequence(rdev,
1045                                                  cedar_mgcg_init,
1046                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
1047                 break;
1048         case CHIP_PALM:
1049                 radeon_program_register_sequence(rdev,
1050                                                  wrestler_golden_registers,
1051                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
1052                 break;
1053         case CHIP_SUMO:
1054                 radeon_program_register_sequence(rdev,
1055                                                  supersumo_golden_registers,
1056                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1057                 break;
1058         case CHIP_SUMO2:
1059                 radeon_program_register_sequence(rdev,
1060                                                  supersumo_golden_registers,
1061                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1062                 radeon_program_register_sequence(rdev,
1063                                                  sumo_golden_registers,
1064                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
1065                 break;
1066         case CHIP_BARTS:
1067                 radeon_program_register_sequence(rdev,
1068                                                  barts_golden_registers,
1069                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
1070                 break;
1071         case CHIP_TURKS:
1072                 radeon_program_register_sequence(rdev,
1073                                                  turks_golden_registers,
1074                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
1075                 break;
1076         case CHIP_CAICOS:
1077                 radeon_program_register_sequence(rdev,
1078                                                  caicos_golden_registers,
1079                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1080                 break;
1081         default:
1082                 break;
1083         }
1084 }
1085
1086 /**
1087  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1088  *
1089  * @rdev: radeon_device pointer
1090  * @reg: register offset in bytes
1091  * @val: register value
1092  *
1093  * Returns 0 for success or -EINVAL for an invalid register
1094  *
1095  */
1096 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1097                                         u32 reg, u32 *val)
1098 {
1099         switch (reg) {
1100         case GRBM_STATUS:
1101         case GRBM_STATUS_SE0:
1102         case GRBM_STATUS_SE1:
1103         case SRBM_STATUS:
1104         case SRBM_STATUS2:
1105         case DMA_STATUS_REG:
1106         case UVD_STATUS:
1107                 *val = RREG32(reg);
1108                 return 0;
1109         default:
1110                 return -EINVAL;
1111         }
1112 }
1113
1114 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1115                              unsigned *bankh, unsigned *mtaspect,
1116                              unsigned *tile_split)
1117 {
1118         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1119         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1120         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1121         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1122         switch (*bankw) {
1123         default:
1124         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1125         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1126         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1127         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1128         }
1129         switch (*bankh) {
1130         default:
1131         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1132         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1133         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1134         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1135         }
1136         switch (*mtaspect) {
1137         default:
1138         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1139         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1140         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1141         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1142         }
1143 }
1144
1145 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1146                               u32 cntl_reg, u32 status_reg)
1147 {
1148         int r, i;
1149         struct atom_clock_dividers dividers;
1150
1151         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1152                                            clock, false, &dividers);
1153         if (r)
1154                 return r;
1155
1156         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1157
1158         for (i = 0; i < 100; i++) {
1159                 if (RREG32(status_reg) & DCLK_STATUS)
1160                         break;
1161                 mdelay(10);
1162         }
1163         if (i == 100)
1164                 return -ETIMEDOUT;
1165
1166         return 0;
1167 }
1168
1169 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1170 {
1171         int r = 0;
1172         u32 cg_scratch = RREG32(CG_SCRATCH1);
1173
1174         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1175         if (r)
1176                 goto done;
1177         cg_scratch &= 0xffff0000;
1178         cg_scratch |= vclk / 100; /* Mhz */
1179
1180         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1181         if (r)
1182                 goto done;
1183         cg_scratch &= 0x0000ffff;
1184         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1185
1186 done:
1187         WREG32(CG_SCRATCH1, cg_scratch);
1188
1189         return r;
1190 }
1191
1192 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1193 {
1194         /* start off with something large */
1195         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1196         int r;
1197
1198         /* bypass vclk and dclk with bclk */
1199         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1200                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1201                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1202
1203         /* put PLL in bypass mode */
1204         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1205
1206         if (!vclk || !dclk) {
1207                 /* keep the Bypass mode, put PLL to sleep */
1208                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1209                 return 0;
1210         }
1211
1212         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1213                                           16384, 0x03FFFFFF, 0, 128, 5,
1214                                           &fb_div, &vclk_div, &dclk_div);
1215         if (r)
1216                 return r;
1217
1218         /* set VCO_MODE to 1 */
1219         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1220
1221         /* toggle UPLL_SLEEP to 1 then back to 0 */
1222         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1223         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1224
1225         /* deassert UPLL_RESET */
1226         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1227
1228         mdelay(1);
1229
1230         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1231         if (r)
1232                 return r;
1233
1234         /* assert UPLL_RESET again */
1235         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1236
1237         /* disable spread spectrum. */
1238         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1239
1240         /* set feedback divider */
1241         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1242
1243         /* set ref divider to 0 */
1244         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1245
1246         if (fb_div < 307200)
1247                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1248         else
1249                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1250
1251         /* set PDIV_A and PDIV_B */
1252         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1253                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1254                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1255
1256         /* give the PLL some time to settle */
1257         mdelay(15);
1258
1259         /* deassert PLL_RESET */
1260         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1261
1262         mdelay(15);
1263
1264         /* switch from bypass mode to normal mode */
1265         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1266
1267         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1268         if (r)
1269                 return r;
1270
1271         /* switch VCLK and DCLK selection */
1272         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1273                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1274                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1275
1276         mdelay(100);
1277
1278         return 0;
1279 }
1280
1281 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1282 {
1283         int readrq;
1284         u16 v;
1285
1286         readrq = pcie_get_readrq(rdev->pdev);
1287         v = ffs(readrq) - 8;
1288         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1289          * to avoid hangs or perfomance issues
1290          */
1291         if ((v == 0) || (v == 6) || (v == 7))
1292                 pcie_set_readrq(rdev->pdev, 512);
1293 }
1294
1295 void dce4_program_fmt(struct drm_encoder *encoder)
1296 {
1297         struct drm_device *dev = encoder->dev;
1298         struct radeon_device *rdev = dev->dev_private;
1299         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1300         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1301         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1302         int bpc = 0;
1303         u32 tmp = 0;
1304         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1305
1306         if (connector) {
1307                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1308                 bpc = radeon_get_monitor_bpc(connector);
1309                 dither = radeon_connector->dither;
1310         }
1311
1312         /* LVDS/eDP FMT is set up by atom */
1313         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1314                 return;
1315
1316         /* not needed for analog */
1317         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1318             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1319                 return;
1320
1321         if (bpc == 0)
1322                 return;
1323
1324         switch (bpc) {
1325         case 6:
1326                 if (dither == RADEON_FMT_DITHER_ENABLE)
1327                         /* XXX sort out optimal dither settings */
1328                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329                                 FMT_SPATIAL_DITHER_EN);
1330                 else
1331                         tmp |= FMT_TRUNCATE_EN;
1332                 break;
1333         case 8:
1334                 if (dither == RADEON_FMT_DITHER_ENABLE)
1335                         /* XXX sort out optimal dither settings */
1336                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1337                                 FMT_RGB_RANDOM_ENABLE |
1338                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1339                 else
1340                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1341                 break;
1342         case 10:
1343         default:
1344                 /* not needed */
1345                 break;
1346         }
1347
1348         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1349 }
1350
1351 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1352 {
1353         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1354                 return true;
1355         else
1356                 return false;
1357 }
1358
1359 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1360 {
1361         u32 pos1, pos2;
1362
1363         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1364         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1365
1366         if (pos1 != pos2)
1367                 return true;
1368         else
1369                 return false;
1370 }
1371
1372 /**
1373  * dce4_wait_for_vblank - vblank wait asic callback.
1374  *
1375  * @rdev: radeon_device pointer
1376  * @crtc: crtc to wait for vblank on
1377  *
1378  * Wait for vblank on the requested crtc (evergreen+).
1379  */
1380 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1381 {
1382         unsigned i = 0;
1383
1384         if (crtc >= rdev->num_crtc)
1385                 return;
1386
1387         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1388                 return;
1389
1390         /* depending on when we hit vblank, we may be close to active; if so,
1391          * wait for another frame.
1392          */
1393         while (dce4_is_in_vblank(rdev, crtc)) {
1394                 if (i++ % 100 == 0) {
1395                         if (!dce4_is_counter_moving(rdev, crtc))
1396                                 break;
1397                 }
1398         }
1399
1400         while (!dce4_is_in_vblank(rdev, crtc)) {
1401                 if (i++ % 100 == 0) {
1402                         if (!dce4_is_counter_moving(rdev, crtc))
1403                                 break;
1404                 }
1405         }
1406 }
1407
1408 /**
1409  * evergreen_page_flip - pageflip callback.
1410  *
1411  * @rdev: radeon_device pointer
1412  * @crtc_id: crtc to cleanup pageflip on
1413  * @crtc_base: new address of the crtc (GPU MC address)
1414  *
1415  * Triggers the actual pageflip by updating the primary
1416  * surface base address (evergreen+).
1417  */
1418 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1419                          bool async)
1420 {
1421         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1422
1423         /* update the scanout addresses */
1424         WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1425                async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1426         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1427                upper_32_bits(crtc_base));
1428         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1429                (u32)crtc_base);
1430         /* post the write */
1431         RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1432 }
1433
1434 /**
1435  * evergreen_page_flip_pending - check if page flip is still pending
1436  *
1437  * @rdev: radeon_device pointer
1438  * @crtc_id: crtc to check
1439  *
1440  * Returns the current update pending status.
1441  */
1442 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1443 {
1444         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1445
1446         /* Return current update_pending status: */
1447         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1448                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1449 }
1450
1451 /* get temperature in millidegrees */
1452 int evergreen_get_temp(struct radeon_device *rdev)
1453 {
1454         u32 temp, toffset;
1455         int actual_temp = 0;
1456
1457         if (rdev->family == CHIP_JUNIPER) {
1458                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1459                         TOFFSET_SHIFT;
1460                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1461                         TS0_ADC_DOUT_SHIFT;
1462
1463                 if (toffset & 0x100)
1464                         actual_temp = temp / 2 - (0x200 - toffset);
1465                 else
1466                         actual_temp = temp / 2 + toffset;
1467
1468                 actual_temp = actual_temp * 1000;
1469
1470         } else {
1471                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1472                         ASIC_T_SHIFT;
1473
1474                 if (temp & 0x400)
1475                         actual_temp = -256;
1476                 else if (temp & 0x200)
1477                         actual_temp = 255;
1478                 else if (temp & 0x100) {
1479                         actual_temp = temp & 0x1ff;
1480                         actual_temp |= ~0x1ff;
1481                 } else
1482                         actual_temp = temp & 0xff;
1483
1484                 actual_temp = (actual_temp * 1000) / 2;
1485         }
1486
1487         return actual_temp;
1488 }
1489
1490 int sumo_get_temp(struct radeon_device *rdev)
1491 {
1492         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1493         int actual_temp = temp - 49;
1494
1495         return actual_temp * 1000;
1496 }
1497
1498 /**
1499  * sumo_pm_init_profile - Initialize power profiles callback.
1500  *
1501  * @rdev: radeon_device pointer
1502  *
1503  * Initialize the power states used in profile mode
1504  * (sumo, trinity, SI).
1505  * Used for profile mode only.
1506  */
1507 void sumo_pm_init_profile(struct radeon_device *rdev)
1508 {
1509         int idx;
1510
1511         /* default */
1512         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1513         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1514         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1515         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1516
1517         /* low,mid sh/mh */
1518         if (rdev->flags & RADEON_IS_MOBILITY)
1519                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1520         else
1521                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1522
1523         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1524         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1525         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1526         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1527
1528         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1529         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1530         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1531         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1532
1533         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1534         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1535         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1536         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1537
1538         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1539         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1540         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1541         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1542
1543         /* high sh/mh */
1544         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1545         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1546         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1547         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1548         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1549                 rdev->pm.power_state[idx].num_clock_modes - 1;
1550
1551         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1552         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1553         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1554         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1555                 rdev->pm.power_state[idx].num_clock_modes - 1;
1556 }
1557
1558 /**
1559  * btc_pm_init_profile - Initialize power profiles callback.
1560  *
1561  * @rdev: radeon_device pointer
1562  *
1563  * Initialize the power states used in profile mode
1564  * (BTC, cayman).
1565  * Used for profile mode only.
1566  */
1567 void btc_pm_init_profile(struct radeon_device *rdev)
1568 {
1569         int idx;
1570
1571         /* default */
1572         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1573         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1574         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1575         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1576         /* starting with BTC, there is one state that is used for both
1577          * MH and SH.  Difference is that we always use the high clock index for
1578          * mclk.
1579          */
1580         if (rdev->flags & RADEON_IS_MOBILITY)
1581                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1582         else
1583                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1584         /* low sh */
1585         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1586         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1587         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1588         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1589         /* mid sh */
1590         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1591         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1592         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1593         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1594         /* high sh */
1595         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1596         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1597         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1598         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1599         /* low mh */
1600         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1601         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1602         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1603         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1604         /* mid mh */
1605         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1606         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1607         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1608         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1609         /* high mh */
1610         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1611         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1612         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1613         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1614 }
1615
1616 /**
1617  * evergreen_pm_misc - set additional pm hw parameters callback.
1618  *
1619  * @rdev: radeon_device pointer
1620  *
1621  * Set non-clock parameters associated with a power state
1622  * (voltage, etc.) (evergreen+).
1623  */
1624 void evergreen_pm_misc(struct radeon_device *rdev)
1625 {
1626         int req_ps_idx = rdev->pm.requested_power_state_index;
1627         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1628         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1629         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1630
1631         if (voltage->type == VOLTAGE_SW) {
1632                 /* 0xff0x are flags rather then an actual voltage */
1633                 if ((voltage->voltage & 0xff00) == 0xff00)
1634                         return;
1635                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1636                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1637                         rdev->pm.current_vddc = voltage->voltage;
1638                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1639                 }
1640
1641                 /* starting with BTC, there is one state that is used for both
1642                  * MH and SH.  Difference is that we always use the high clock index for
1643                  * mclk and vddci.
1644                  */
1645                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1646                     (rdev->family >= CHIP_BARTS) &&
1647                     rdev->pm.active_crtc_count &&
1648                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1649                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1650                         voltage = &rdev->pm.power_state[req_ps_idx].
1651                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1652
1653                 /* 0xff0x are flags rather then an actual voltage */
1654                 if ((voltage->vddci & 0xff00) == 0xff00)
1655                         return;
1656                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1657                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1658                         rdev->pm.current_vddci = voltage->vddci;
1659                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1660                 }
1661         }
1662 }
1663
1664 /**
1665  * evergreen_pm_prepare - pre-power state change callback.
1666  *
1667  * @rdev: radeon_device pointer
1668  *
1669  * Prepare for a power state change (evergreen+).
1670  */
1671 void evergreen_pm_prepare(struct radeon_device *rdev)
1672 {
1673         struct drm_device *ddev = rdev->ddev;
1674         struct drm_crtc *crtc;
1675         struct radeon_crtc *radeon_crtc;
1676         u32 tmp;
1677
1678         /* disable any active CRTCs */
1679         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1680                 radeon_crtc = to_radeon_crtc(crtc);
1681                 if (radeon_crtc->enabled) {
1682                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1683                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1684                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1685                 }
1686         }
1687 }
1688
1689 /**
1690  * evergreen_pm_finish - post-power state change callback.
1691  *
1692  * @rdev: radeon_device pointer
1693  *
1694  * Clean up after a power state change (evergreen+).
1695  */
1696 void evergreen_pm_finish(struct radeon_device *rdev)
1697 {
1698         struct drm_device *ddev = rdev->ddev;
1699         struct drm_crtc *crtc;
1700         struct radeon_crtc *radeon_crtc;
1701         u32 tmp;
1702
1703         /* enable any active CRTCs */
1704         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1705                 radeon_crtc = to_radeon_crtc(crtc);
1706                 if (radeon_crtc->enabled) {
1707                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1708                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1709                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1710                 }
1711         }
1712 }
1713
1714 /**
1715  * evergreen_hpd_sense - hpd sense callback.
1716  *
1717  * @rdev: radeon_device pointer
1718  * @hpd: hpd (hotplug detect) pin
1719  *
1720  * Checks if a digital monitor is connected (evergreen+).
1721  * Returns true if connected, false if not connected.
1722  */
1723 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1724 {
1725         if (hpd == RADEON_HPD_NONE)
1726                 return false;
1727
1728         return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1729 }
1730
1731 /**
1732  * evergreen_hpd_set_polarity - hpd set polarity callback.
1733  *
1734  * @rdev: radeon_device pointer
1735  * @hpd: hpd (hotplug detect) pin
1736  *
1737  * Set the polarity of the hpd pin (evergreen+).
1738  */
1739 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1740                                 enum radeon_hpd_id hpd)
1741 {
1742         bool connected = evergreen_hpd_sense(rdev, hpd);
1743
1744         if (hpd == RADEON_HPD_NONE)
1745                 return;
1746
1747         if (connected)
1748                 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1749         else
1750                 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1751 }
1752
1753 /**
1754  * evergreen_hpd_init - hpd setup callback.
1755  *
1756  * @rdev: radeon_device pointer
1757  *
1758  * Setup the hpd pins used by the card (evergreen+).
1759  * Enable the pin, set the polarity, and enable the hpd interrupts.
1760  */
1761 void evergreen_hpd_init(struct radeon_device *rdev)
1762 {
1763         struct drm_device *dev = rdev->ddev;
1764         struct drm_connector *connector;
1765         unsigned enabled = 0;
1766         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1767                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1768
1769         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1770                 enum radeon_hpd_id hpd =
1771                         to_radeon_connector(connector)->hpd.hpd;
1772
1773                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1774                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1775                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1776                          * aux dp channel on imac and help (but not completely fix)
1777                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1778                          * also avoid interrupt storms during dpms.
1779                          */
1780                         continue;
1781                 }
1782
1783                 if (hpd == RADEON_HPD_NONE)
1784                         continue;
1785
1786                 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1787                 enabled |= 1 << hpd;
1788
1789                 radeon_hpd_set_polarity(rdev, hpd);
1790         }
1791         radeon_irq_kms_enable_hpd(rdev, enabled);
1792 }
1793
1794 /**
1795  * evergreen_hpd_fini - hpd tear down callback.
1796  *
1797  * @rdev: radeon_device pointer
1798  *
1799  * Tear down the hpd pins used by the card (evergreen+).
1800  * Disable the hpd interrupts.
1801  */
1802 void evergreen_hpd_fini(struct radeon_device *rdev)
1803 {
1804         struct drm_device *dev = rdev->ddev;
1805         struct drm_connector *connector;
1806         unsigned disabled = 0;
1807
1808         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1809                 enum radeon_hpd_id hpd =
1810                         to_radeon_connector(connector)->hpd.hpd;
1811
1812                 if (hpd == RADEON_HPD_NONE)
1813                         continue;
1814
1815                 WREG32(DC_HPDx_CONTROL(hpd), 0);
1816                 disabled |= 1 << hpd;
1817         }
1818         radeon_irq_kms_disable_hpd(rdev, disabled);
1819 }
1820
1821 /* watermark setup */
1822
1823 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1824                                         struct radeon_crtc *radeon_crtc,
1825                                         struct drm_display_mode *mode,
1826                                         struct drm_display_mode *other_mode)
1827 {
1828         u32 tmp, buffer_alloc, i;
1829         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1830         /*
1831          * Line Buffer Setup
1832          * There are 3 line buffers, each one shared by 2 display controllers.
1833          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1834          * the display controllers.  The paritioning is done via one of four
1835          * preset allocations specified in bits 2:0:
1836          * first display controller
1837          *  0 - first half of lb (3840 * 2)
1838          *  1 - first 3/4 of lb (5760 * 2)
1839          *  2 - whole lb (7680 * 2), other crtc must be disabled
1840          *  3 - first 1/4 of lb (1920 * 2)
1841          * second display controller
1842          *  4 - second half of lb (3840 * 2)
1843          *  5 - second 3/4 of lb (5760 * 2)
1844          *  6 - whole lb (7680 * 2), other crtc must be disabled
1845          *  7 - last 1/4 of lb (1920 * 2)
1846          */
1847         /* this can get tricky if we have two large displays on a paired group
1848          * of crtcs.  Ideally for multiple large displays we'd assign them to
1849          * non-linked crtcs for maximum line buffer allocation.
1850          */
1851         if (radeon_crtc->base.enabled && mode) {
1852                 if (other_mode) {
1853                         tmp = 0; /* 1/2 */
1854                         buffer_alloc = 1;
1855                 } else {
1856                         tmp = 2; /* whole */
1857                         buffer_alloc = 2;
1858                 }
1859         } else {
1860                 tmp = 0;
1861                 buffer_alloc = 0;
1862         }
1863
1864         /* second controller of the pair uses second half of the lb */
1865         if (radeon_crtc->crtc_id % 2)
1866                 tmp += 4;
1867         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1868
1869         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1870                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1871                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1872                 for (i = 0; i < rdev->usec_timeout; i++) {
1873                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1874                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1875                                 break;
1876                         udelay(1);
1877                 }
1878         }
1879
1880         if (radeon_crtc->base.enabled && mode) {
1881                 switch (tmp) {
1882                 case 0:
1883                 case 4:
1884                 default:
1885                         if (ASIC_IS_DCE5(rdev))
1886                                 return 4096 * 2;
1887                         else
1888                                 return 3840 * 2;
1889                 case 1:
1890                 case 5:
1891                         if (ASIC_IS_DCE5(rdev))
1892                                 return 6144 * 2;
1893                         else
1894                                 return 5760 * 2;
1895                 case 2:
1896                 case 6:
1897                         if (ASIC_IS_DCE5(rdev))
1898                                 return 8192 * 2;
1899                         else
1900                                 return 7680 * 2;
1901                 case 3:
1902                 case 7:
1903                         if (ASIC_IS_DCE5(rdev))
1904                                 return 2048 * 2;
1905                         else
1906                                 return 1920 * 2;
1907                 }
1908         }
1909
1910         /* controller not enabled, so no lb used */
1911         return 0;
1912 }
1913
1914 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1915 {
1916         u32 tmp = RREG32(MC_SHARED_CHMAP);
1917
1918         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1919         case 0:
1920         default:
1921                 return 1;
1922         case 1:
1923                 return 2;
1924         case 2:
1925                 return 4;
1926         case 3:
1927                 return 8;
1928         }
1929 }
1930
1931 struct evergreen_wm_params {
1932         u32 dram_channels; /* number of dram channels */
1933         u32 yclk;          /* bandwidth per dram data pin in kHz */
1934         u32 sclk;          /* engine clock in kHz */
1935         u32 disp_clk;      /* display clock in kHz */
1936         u32 src_width;     /* viewport width */
1937         u32 active_time;   /* active display time in ns */
1938         u32 blank_time;    /* blank time in ns */
1939         bool interlaced;    /* mode is interlaced */
1940         fixed20_12 vsc;    /* vertical scale ratio */
1941         u32 num_heads;     /* number of active crtcs */
1942         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1943         u32 lb_size;       /* line buffer allocated to pipe */
1944         u32 vtaps;         /* vertical scaler taps */
1945 };
1946
1947 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1948 {
1949         /* Calculate DRAM Bandwidth and the part allocated to display. */
1950         fixed20_12 dram_efficiency; /* 0.7 */
1951         fixed20_12 yclk, dram_channels, bandwidth;
1952         fixed20_12 a;
1953
1954         a.full = dfixed_const(1000);
1955         yclk.full = dfixed_const(wm->yclk);
1956         yclk.full = dfixed_div(yclk, a);
1957         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1958         a.full = dfixed_const(10);
1959         dram_efficiency.full = dfixed_const(7);
1960         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1961         bandwidth.full = dfixed_mul(dram_channels, yclk);
1962         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1963
1964         return dfixed_trunc(bandwidth);
1965 }
1966
1967 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1968 {
1969         /* Calculate DRAM Bandwidth and the part allocated to display. */
1970         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1971         fixed20_12 yclk, dram_channels, bandwidth;
1972         fixed20_12 a;
1973
1974         a.full = dfixed_const(1000);
1975         yclk.full = dfixed_const(wm->yclk);
1976         yclk.full = dfixed_div(yclk, a);
1977         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1978         a.full = dfixed_const(10);
1979         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1980         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1981         bandwidth.full = dfixed_mul(dram_channels, yclk);
1982         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1983
1984         return dfixed_trunc(bandwidth);
1985 }
1986
1987 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1988 {
1989         /* Calculate the display Data return Bandwidth */
1990         fixed20_12 return_efficiency; /* 0.8 */
1991         fixed20_12 sclk, bandwidth;
1992         fixed20_12 a;
1993
1994         a.full = dfixed_const(1000);
1995         sclk.full = dfixed_const(wm->sclk);
1996         sclk.full = dfixed_div(sclk, a);
1997         a.full = dfixed_const(10);
1998         return_efficiency.full = dfixed_const(8);
1999         return_efficiency.full = dfixed_div(return_efficiency, a);
2000         a.full = dfixed_const(32);
2001         bandwidth.full = dfixed_mul(a, sclk);
2002         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2003
2004         return dfixed_trunc(bandwidth);
2005 }
2006
2007 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2008 {
2009         /* Calculate the DMIF Request Bandwidth */
2010         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2011         fixed20_12 disp_clk, bandwidth;
2012         fixed20_12 a;
2013
2014         a.full = dfixed_const(1000);
2015         disp_clk.full = dfixed_const(wm->disp_clk);
2016         disp_clk.full = dfixed_div(disp_clk, a);
2017         a.full = dfixed_const(10);
2018         disp_clk_request_efficiency.full = dfixed_const(8);
2019         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2020         a.full = dfixed_const(32);
2021         bandwidth.full = dfixed_mul(a, disp_clk);
2022         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2023
2024         return dfixed_trunc(bandwidth);
2025 }
2026
2027 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2028 {
2029         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2030         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2031         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2032         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2033
2034         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2035 }
2036
2037 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2038 {
2039         /* Calculate the display mode Average Bandwidth
2040          * DisplayMode should contain the source and destination dimensions,
2041          * timing, etc.
2042          */
2043         fixed20_12 bpp;
2044         fixed20_12 line_time;
2045         fixed20_12 src_width;
2046         fixed20_12 bandwidth;
2047         fixed20_12 a;
2048
2049         a.full = dfixed_const(1000);
2050         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2051         line_time.full = dfixed_div(line_time, a);
2052         bpp.full = dfixed_const(wm->bytes_per_pixel);
2053         src_width.full = dfixed_const(wm->src_width);
2054         bandwidth.full = dfixed_mul(src_width, bpp);
2055         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2056         bandwidth.full = dfixed_div(bandwidth, line_time);
2057
2058         return dfixed_trunc(bandwidth);
2059 }
2060
2061 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2062 {
2063         /* First calcualte the latency in ns */
2064         u32 mc_latency = 2000; /* 2000 ns. */
2065         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2066         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2067         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2068         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2069         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2070                 (wm->num_heads * cursor_line_pair_return_time);
2071         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2072         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2073         fixed20_12 a, b, c;
2074
2075         if (wm->num_heads == 0)
2076                 return 0;
2077
2078         a.full = dfixed_const(2);
2079         b.full = dfixed_const(1);
2080         if ((wm->vsc.full > a.full) ||
2081             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2082             (wm->vtaps >= 5) ||
2083             ((wm->vsc.full >= a.full) && wm->interlaced))
2084                 max_src_lines_per_dst_line = 4;
2085         else
2086                 max_src_lines_per_dst_line = 2;
2087
2088         a.full = dfixed_const(available_bandwidth);
2089         b.full = dfixed_const(wm->num_heads);
2090         a.full = dfixed_div(a, b);
2091
2092         lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2093
2094         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2095         b.full = dfixed_const(1000);
2096         c.full = dfixed_const(lb_fill_bw);
2097         b.full = dfixed_div(c, b);
2098         a.full = dfixed_div(a, b);
2099         line_fill_time = dfixed_trunc(a);
2100
2101         if (line_fill_time < wm->active_time)
2102                 return latency;
2103         else
2104                 return latency + (line_fill_time - wm->active_time);
2105
2106 }
2107
2108 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2109 {
2110         if (evergreen_average_bandwidth(wm) <=
2111             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2112                 return true;
2113         else
2114                 return false;
2115 };
2116
2117 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2118 {
2119         if (evergreen_average_bandwidth(wm) <=
2120             (evergreen_available_bandwidth(wm) / wm->num_heads))
2121                 return true;
2122         else
2123                 return false;
2124 };
2125
2126 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2127 {
2128         u32 lb_partitions = wm->lb_size / wm->src_width;
2129         u32 line_time = wm->active_time + wm->blank_time;
2130         u32 latency_tolerant_lines;
2131         u32 latency_hiding;
2132         fixed20_12 a;
2133
2134         a.full = dfixed_const(1);
2135         if (wm->vsc.full > a.full)
2136                 latency_tolerant_lines = 1;
2137         else {
2138                 if (lb_partitions <= (wm->vtaps + 1))
2139                         latency_tolerant_lines = 1;
2140                 else
2141                         latency_tolerant_lines = 2;
2142         }
2143
2144         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2145
2146         if (evergreen_latency_watermark(wm) <= latency_hiding)
2147                 return true;
2148         else
2149                 return false;
2150 }
2151
2152 static void evergreen_program_watermarks(struct radeon_device *rdev,
2153                                          struct radeon_crtc *radeon_crtc,
2154                                          u32 lb_size, u32 num_heads)
2155 {
2156         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2157         struct evergreen_wm_params wm_low, wm_high;
2158         u32 dram_channels;
2159         u32 active_time;
2160         u32 line_time = 0;
2161         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2162         u32 priority_a_mark = 0, priority_b_mark = 0;
2163         u32 priority_a_cnt = PRIORITY_OFF;
2164         u32 priority_b_cnt = PRIORITY_OFF;
2165         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2166         u32 tmp, arb_control3;
2167         fixed20_12 a, b, c;
2168
2169         if (radeon_crtc->base.enabled && num_heads && mode) {
2170                 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2171                                             (u32)mode->clock);
2172                 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2173                                           (u32)mode->clock);
2174                 line_time = min(line_time, (u32)65535);
2175                 priority_a_cnt = 0;
2176                 priority_b_cnt = 0;
2177                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2178
2179                 /* watermark for high clocks */
2180                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2181                         wm_high.yclk =
2182                                 radeon_dpm_get_mclk(rdev, false) * 10;
2183                         wm_high.sclk =
2184                                 radeon_dpm_get_sclk(rdev, false) * 10;
2185                 } else {
2186                         wm_high.yclk = rdev->pm.current_mclk * 10;
2187                         wm_high.sclk = rdev->pm.current_sclk * 10;
2188                 }
2189
2190                 wm_high.disp_clk = mode->clock;
2191                 wm_high.src_width = mode->crtc_hdisplay;
2192                 wm_high.active_time = active_time;
2193                 wm_high.blank_time = line_time - wm_high.active_time;
2194                 wm_high.interlaced = false;
2195                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2196                         wm_high.interlaced = true;
2197                 wm_high.vsc = radeon_crtc->vsc;
2198                 wm_high.vtaps = 1;
2199                 if (radeon_crtc->rmx_type != RMX_OFF)
2200                         wm_high.vtaps = 2;
2201                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2202                 wm_high.lb_size = lb_size;
2203                 wm_high.dram_channels = dram_channels;
2204                 wm_high.num_heads = num_heads;
2205
2206                 /* watermark for low clocks */
2207                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2208                         wm_low.yclk =
2209                                 radeon_dpm_get_mclk(rdev, true) * 10;
2210                         wm_low.sclk =
2211                                 radeon_dpm_get_sclk(rdev, true) * 10;
2212                 } else {
2213                         wm_low.yclk = rdev->pm.current_mclk * 10;
2214                         wm_low.sclk = rdev->pm.current_sclk * 10;
2215                 }
2216
2217                 wm_low.disp_clk = mode->clock;
2218                 wm_low.src_width = mode->crtc_hdisplay;
2219                 wm_low.active_time = active_time;
2220                 wm_low.blank_time = line_time - wm_low.active_time;
2221                 wm_low.interlaced = false;
2222                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2223                         wm_low.interlaced = true;
2224                 wm_low.vsc = radeon_crtc->vsc;
2225                 wm_low.vtaps = 1;
2226                 if (radeon_crtc->rmx_type != RMX_OFF)
2227                         wm_low.vtaps = 2;
2228                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2229                 wm_low.lb_size = lb_size;
2230                 wm_low.dram_channels = dram_channels;
2231                 wm_low.num_heads = num_heads;
2232
2233                 /* set for high clocks */
2234                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2235                 /* set for low clocks */
2236                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2237
2238                 /* possibly force display priority to high */
2239                 /* should really do this at mode validation time... */
2240                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2241                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2242                     !evergreen_check_latency_hiding(&wm_high) ||
2243                     (rdev->disp_priority == 2)) {
2244                         DRM_DEBUG_KMS("force priority a to high\n");
2245                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2246                 }
2247                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2248                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2249                     !evergreen_check_latency_hiding(&wm_low) ||
2250                     (rdev->disp_priority == 2)) {
2251                         DRM_DEBUG_KMS("force priority b to high\n");
2252                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2253                 }
2254
2255                 a.full = dfixed_const(1000);
2256                 b.full = dfixed_const(mode->clock);
2257                 b.full = dfixed_div(b, a);
2258                 c.full = dfixed_const(latency_watermark_a);
2259                 c.full = dfixed_mul(c, b);
2260                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2261                 c.full = dfixed_div(c, a);
2262                 a.full = dfixed_const(16);
2263                 c.full = dfixed_div(c, a);
2264                 priority_a_mark = dfixed_trunc(c);
2265                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2266
2267                 a.full = dfixed_const(1000);
2268                 b.full = dfixed_const(mode->clock);
2269                 b.full = dfixed_div(b, a);
2270                 c.full = dfixed_const(latency_watermark_b);
2271                 c.full = dfixed_mul(c, b);
2272                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2273                 c.full = dfixed_div(c, a);
2274                 a.full = dfixed_const(16);
2275                 c.full = dfixed_div(c, a);
2276                 priority_b_mark = dfixed_trunc(c);
2277                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2278
2279                 /* Save number of lines the linebuffer leads before the scanout */
2280                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2281         }
2282
2283         /* select wm A */
2284         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2285         tmp = arb_control3;
2286         tmp &= ~LATENCY_WATERMARK_MASK(3);
2287         tmp |= LATENCY_WATERMARK_MASK(1);
2288         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2289         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2290                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2291                 LATENCY_HIGH_WATERMARK(line_time)));
2292         /* select wm B */
2293         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2294         tmp &= ~LATENCY_WATERMARK_MASK(3);
2295         tmp |= LATENCY_WATERMARK_MASK(2);
2296         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2297         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2298                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2299                 LATENCY_HIGH_WATERMARK(line_time)));
2300         /* restore original selection */
2301         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2302
2303         /* write the priority marks */
2304         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2305         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2306
2307         /* save values for DPM */
2308         radeon_crtc->line_time = line_time;
2309         radeon_crtc->wm_high = latency_watermark_a;
2310         radeon_crtc->wm_low = latency_watermark_b;
2311 }
2312
2313 /**
2314  * evergreen_bandwidth_update - update display watermarks callback.
2315  *
2316  * @rdev: radeon_device pointer
2317  *
2318  * Update the display watermarks based on the requested mode(s)
2319  * (evergreen+).
2320  */
2321 void evergreen_bandwidth_update(struct radeon_device *rdev)
2322 {
2323         struct drm_display_mode *mode0 = NULL;
2324         struct drm_display_mode *mode1 = NULL;
2325         u32 num_heads = 0, lb_size;
2326         int i;
2327
2328         if (!rdev->mode_info.mode_config_initialized)
2329                 return;
2330
2331         radeon_update_display_priority(rdev);
2332
2333         for (i = 0; i < rdev->num_crtc; i++) {
2334                 if (rdev->mode_info.crtcs[i]->base.enabled)
2335                         num_heads++;
2336         }
2337         for (i = 0; i < rdev->num_crtc; i += 2) {
2338                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2339                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2340                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2341                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2342                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2343                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2344         }
2345 }
2346
2347 /**
2348  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2349  *
2350  * @rdev: radeon_device pointer
2351  *
2352  * Wait for the MC (memory controller) to be idle.
2353  * (evergreen+).
2354  * Returns 0 if the MC is idle, -1 if not.
2355  */
2356 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2357 {
2358         unsigned i;
2359         u32 tmp;
2360
2361         for (i = 0; i < rdev->usec_timeout; i++) {
2362                 /* read MC_STATUS */
2363                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2364                 if (!tmp)
2365                         return 0;
2366                 udelay(1);
2367         }
2368         return -1;
2369 }
2370
2371 /*
2372  * GART
2373  */
2374 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2375 {
2376         unsigned i;
2377         u32 tmp;
2378
2379         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2380
2381         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2382         for (i = 0; i < rdev->usec_timeout; i++) {
2383                 /* read MC_STATUS */
2384                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2385                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2386                 if (tmp == 2) {
2387                         pr_warn("[drm] r600 flush TLB failed\n");
2388                         return;
2389                 }
2390                 if (tmp) {
2391                         return;
2392                 }
2393                 udelay(1);
2394         }
2395 }
2396
2397 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2398 {
2399         u32 tmp;
2400         int r;
2401
2402         if (rdev->gart.robj == NULL) {
2403                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2404                 return -EINVAL;
2405         }
2406         r = radeon_gart_table_vram_pin(rdev);
2407         if (r)
2408                 return r;
2409         /* Setup L2 cache */
2410         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2411                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2412                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2413         WREG32(VM_L2_CNTL2, 0);
2414         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2415         /* Setup TLB control */
2416         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2417                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2418                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2419                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2420         if (rdev->flags & RADEON_IS_IGP) {
2421                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2422                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2423                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2424         } else {
2425                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2426                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2427                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2428                 if ((rdev->family == CHIP_JUNIPER) ||
2429                     (rdev->family == CHIP_CYPRESS) ||
2430                     (rdev->family == CHIP_HEMLOCK) ||
2431                     (rdev->family == CHIP_BARTS))
2432                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2433         }
2434         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2435         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2436         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2437         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2438         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2439         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2440         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2441         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2442                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2443         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2444                         (u32)(rdev->dummy_page.addr >> 12));
2445         WREG32(VM_CONTEXT1_CNTL, 0);
2446
2447         evergreen_pcie_gart_tlb_flush(rdev);
2448         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2449                  (unsigned)(rdev->mc.gtt_size >> 20),
2450                  (unsigned long long)rdev->gart.table_addr);
2451         rdev->gart.ready = true;
2452         return 0;
2453 }
2454
2455 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2456 {
2457         u32 tmp;
2458
2459         /* Disable all tables */
2460         WREG32(VM_CONTEXT0_CNTL, 0);
2461         WREG32(VM_CONTEXT1_CNTL, 0);
2462
2463         /* Setup L2 cache */
2464         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2465                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2466         WREG32(VM_L2_CNTL2, 0);
2467         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2468         /* Setup TLB control */
2469         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2470         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2471         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2472         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2473         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2474         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2475         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2476         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2477         radeon_gart_table_vram_unpin(rdev);
2478 }
2479
2480 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2481 {
2482         evergreen_pcie_gart_disable(rdev);
2483         radeon_gart_table_vram_free(rdev);
2484         radeon_gart_fini(rdev);
2485 }
2486
2487
2488 static void evergreen_agp_enable(struct radeon_device *rdev)
2489 {
2490         u32 tmp;
2491
2492         /* Setup L2 cache */
2493         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2494                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2495                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2496         WREG32(VM_L2_CNTL2, 0);
2497         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2498         /* Setup TLB control */
2499         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2500                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2501                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2502                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2503         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2504         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2505         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2506         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2507         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2508         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2509         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2510         WREG32(VM_CONTEXT0_CNTL, 0);
2511         WREG32(VM_CONTEXT1_CNTL, 0);
2512 }
2513
2514 static const unsigned ni_dig_offsets[] =
2515 {
2516         NI_DIG0_REGISTER_OFFSET,
2517         NI_DIG1_REGISTER_OFFSET,
2518         NI_DIG2_REGISTER_OFFSET,
2519         NI_DIG3_REGISTER_OFFSET,
2520         NI_DIG4_REGISTER_OFFSET,
2521         NI_DIG5_REGISTER_OFFSET
2522 };
2523
2524 static const unsigned ni_tx_offsets[] =
2525 {
2526         NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2527         NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2528         NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2529         NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2530         NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2531         NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2532 };
2533
2534 static const unsigned evergreen_dp_offsets[] =
2535 {
2536         EVERGREEN_DP0_REGISTER_OFFSET,
2537         EVERGREEN_DP1_REGISTER_OFFSET,
2538         EVERGREEN_DP2_REGISTER_OFFSET,
2539         EVERGREEN_DP3_REGISTER_OFFSET,
2540         EVERGREEN_DP4_REGISTER_OFFSET,
2541         EVERGREEN_DP5_REGISTER_OFFSET
2542 };
2543
2544 static const unsigned evergreen_disp_int_status[] =
2545 {
2546         DISP_INTERRUPT_STATUS,
2547         DISP_INTERRUPT_STATUS_CONTINUE,
2548         DISP_INTERRUPT_STATUS_CONTINUE2,
2549         DISP_INTERRUPT_STATUS_CONTINUE3,
2550         DISP_INTERRUPT_STATUS_CONTINUE4,
2551         DISP_INTERRUPT_STATUS_CONTINUE5
2552 };
2553
2554 /*
2555  * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2556  * We go from crtc to connector and it is not relible  since it
2557  * should be an opposite direction .If crtc is enable then
2558  * find the dig_fe which selects this crtc and insure that it enable.
2559  * if such dig_fe is found then find dig_be which selects found dig_be and
2560  * insure that it enable and in DP_SST mode.
2561  * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2562  * from dp symbols clocks .
2563  */
2564 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2565                                                unsigned crtc_id, unsigned *ret_dig_fe)
2566 {
2567         unsigned i;
2568         unsigned dig_fe;
2569         unsigned dig_be;
2570         unsigned dig_en_be;
2571         unsigned uniphy_pll;
2572         unsigned digs_fe_selected;
2573         unsigned dig_be_mode;
2574         unsigned dig_fe_mask;
2575         bool is_enabled = false;
2576         bool found_crtc = false;
2577
2578         /* loop through all running dig_fe to find selected crtc */
2579         for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2580                 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2581                 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2582                     crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2583                         /* found running pipe */
2584                         found_crtc = true;
2585                         dig_fe_mask = 1 << i;
2586                         dig_fe = i;
2587                         break;
2588                 }
2589         }
2590
2591         if (found_crtc) {
2592                 /* loop through all running dig_be to find selected dig_fe */
2593                 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2594                         dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2595                         /* if dig_fe_selected by dig_be? */
2596                         digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2597                         dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2598                         if (dig_fe_mask &  digs_fe_selected &&
2599                             /* if dig_be in sst mode? */
2600                             dig_be_mode == NI_DIG_BE_DPSST) {
2601                                 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2602                                                    ni_dig_offsets[i]);
2603                                 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2604                                                     ni_tx_offsets[i]);
2605                                 /* dig_be enable and tx is running */
2606                                 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2607                                     dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2608                                     uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2609                                         is_enabled = true;
2610                                         *ret_dig_fe = dig_fe;
2611                                         break;
2612                                 }
2613                         }
2614                 }
2615         }
2616
2617         return is_enabled;
2618 }
2619
2620 /*
2621  * Blank dig when in dp sst mode
2622  * Dig ignores crtc timing
2623  */
2624 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2625                                       unsigned dig_fe)
2626 {
2627         unsigned stream_ctrl;
2628         unsigned fifo_ctrl;
2629         unsigned counter = 0;
2630
2631         if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2632                 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2633                 return;
2634         }
2635
2636         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2637                              evergreen_dp_offsets[dig_fe]);
2638         if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2639                 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2640                 return;
2641         }
2642
2643         stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2644         WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2645                evergreen_dp_offsets[dig_fe], stream_ctrl);
2646
2647         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648                              evergreen_dp_offsets[dig_fe]);
2649         while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2650                 msleep(1);
2651                 counter++;
2652                 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2653                                      evergreen_dp_offsets[dig_fe]);
2654         }
2655         if (counter >= 32 )
2656                 DRM_ERROR("counter exceeds %d\n", counter);
2657
2658         fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2659         fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2660         WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2661
2662 }
2663
2664 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2665 {
2666         u32 crtc_enabled, tmp, frame_count, blackout;
2667         int i, j;
2668         unsigned dig_fe;
2669
2670         if (!ASIC_IS_NODCE(rdev)) {
2671                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2672                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2673
2674                 /* disable VGA render */
2675                 WREG32(VGA_RENDER_CONTROL, 0);
2676         }
2677         /* blank the display controllers */
2678         for (i = 0; i < rdev->num_crtc; i++) {
2679                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2680                 if (crtc_enabled) {
2681                         save->crtc_enabled[i] = true;
2682                         if (ASIC_IS_DCE6(rdev)) {
2683                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2684                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2685                                         radeon_wait_for_vblank(rdev, i);
2686                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2687                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2688                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2689                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2690                                 }
2691                         } else {
2692                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2693                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2694                                         radeon_wait_for_vblank(rdev, i);
2695                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2696                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2697                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2698                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2699                                 }
2700                         }
2701                         /* wait for the next frame */
2702                         frame_count = radeon_get_vblank_counter(rdev, i);
2703                         for (j = 0; j < rdev->usec_timeout; j++) {
2704                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2705                                         break;
2706                                 udelay(1);
2707                         }
2708                         /*we should disable dig if it drives dp sst*/
2709                         /*but we are in radeon_device_init and the topology is unknown*/
2710                         /*and it is available after radeon_modeset_init*/
2711                         /*the following method radeon_atom_encoder_dpms_dig*/
2712                         /*does the job if we initialize it properly*/
2713                         /*for now we do it this manually*/
2714                         /**/
2715                         if (ASIC_IS_DCE5(rdev) &&
2716                             evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2717                                 evergreen_blank_dp_output(rdev, dig_fe);
2718                         /*we could remove 6 lines below*/
2719                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2720                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2721                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2722                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2723                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2724                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2725                         save->crtc_enabled[i] = false;
2726                         /* ***** */
2727                 } else {
2728                         save->crtc_enabled[i] = false;
2729                 }
2730         }
2731
2732         radeon_mc_wait_for_idle(rdev);
2733
2734         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2735         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2736                 /* Block CPU access */
2737                 WREG32(BIF_FB_EN, 0);
2738                 /* blackout the MC */
2739                 blackout &= ~BLACKOUT_MODE_MASK;
2740                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2741         }
2742         /* wait for the MC to settle */
2743         udelay(100);
2744
2745         /* lock double buffered regs */
2746         for (i = 0; i < rdev->num_crtc; i++) {
2747                 if (save->crtc_enabled[i]) {
2748                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2749                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2750                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2751                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2752                         }
2753                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2754                         if (!(tmp & 1)) {
2755                                 tmp |= 1;
2756                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2757                         }
2758                 }
2759         }
2760 }
2761
2762 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2763 {
2764         u32 tmp, frame_count;
2765         int i, j;
2766
2767         /* update crtc base addresses */
2768         for (i = 0; i < rdev->num_crtc; i++) {
2769                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2770                        upper_32_bits(rdev->mc.vram_start));
2771                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2772                        upper_32_bits(rdev->mc.vram_start));
2773                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2774                        (u32)rdev->mc.vram_start);
2775                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2776                        (u32)rdev->mc.vram_start);
2777         }
2778
2779         if (!ASIC_IS_NODCE(rdev)) {
2780                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2781                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2782         }
2783
2784         /* unlock regs and wait for update */
2785         for (i = 0; i < rdev->num_crtc; i++) {
2786                 if (save->crtc_enabled[i]) {
2787                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2788                         if ((tmp & 0x7) != 0) {
2789                                 tmp &= ~0x7;
2790                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2791                         }
2792                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2793                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2794                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2795                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2796                         }
2797                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2798                         if (tmp & 1) {
2799                                 tmp &= ~1;
2800                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2801                         }
2802                         for (j = 0; j < rdev->usec_timeout; j++) {
2803                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2804                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2805                                         break;
2806                                 udelay(1);
2807                         }
2808                 }
2809         }
2810
2811         /* unblackout the MC */
2812         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2813         tmp &= ~BLACKOUT_MODE_MASK;
2814         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2815         /* allow CPU access */
2816         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2817
2818         for (i = 0; i < rdev->num_crtc; i++) {
2819                 if (save->crtc_enabled[i]) {
2820                         if (ASIC_IS_DCE6(rdev)) {
2821                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2822                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2823                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2824                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2825                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2826                         } else {
2827                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2828                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2829                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2830                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2831                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2832                         }
2833                         /* wait for the next frame */
2834                         frame_count = radeon_get_vblank_counter(rdev, i);
2835                         for (j = 0; j < rdev->usec_timeout; j++) {
2836                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2837                                         break;
2838                                 udelay(1);
2839                         }
2840                 }
2841         }
2842         if (!ASIC_IS_NODCE(rdev)) {
2843                 /* Unlock vga access */
2844                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2845                 mdelay(1);
2846                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2847         }
2848 }
2849
2850 void evergreen_mc_program(struct radeon_device *rdev)
2851 {
2852         struct evergreen_mc_save save;
2853         u32 tmp;
2854         int i, j;
2855
2856         /* Initialize HDP */
2857         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2858                 WREG32((0x2c14 + j), 0x00000000);
2859                 WREG32((0x2c18 + j), 0x00000000);
2860                 WREG32((0x2c1c + j), 0x00000000);
2861                 WREG32((0x2c20 + j), 0x00000000);
2862                 WREG32((0x2c24 + j), 0x00000000);
2863         }
2864         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2865
2866         evergreen_mc_stop(rdev, &save);
2867         if (evergreen_mc_wait_for_idle(rdev)) {
2868                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2869         }
2870         /* Lockout access through VGA aperture*/
2871         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2872         /* Update configuration */
2873         if (rdev->flags & RADEON_IS_AGP) {
2874                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2875                         /* VRAM before AGP */
2876                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2877                                 rdev->mc.vram_start >> 12);
2878                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2879                                 rdev->mc.gtt_end >> 12);
2880                 } else {
2881                         /* VRAM after AGP */
2882                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2883                                 rdev->mc.gtt_start >> 12);
2884                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2885                                 rdev->mc.vram_end >> 12);
2886                 }
2887         } else {
2888                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2889                         rdev->mc.vram_start >> 12);
2890                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2891                         rdev->mc.vram_end >> 12);
2892         }
2893         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2894         /* llano/ontario only */
2895         if ((rdev->family == CHIP_PALM) ||
2896             (rdev->family == CHIP_SUMO) ||
2897             (rdev->family == CHIP_SUMO2)) {
2898                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2899                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2900                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2901                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2902         }
2903         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2904         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2905         WREG32(MC_VM_FB_LOCATION, tmp);
2906         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2907         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2908         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2909         if (rdev->flags & RADEON_IS_AGP) {
2910                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2911                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2912                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2913         } else {
2914                 WREG32(MC_VM_AGP_BASE, 0);
2915                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2916                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2917         }
2918         if (evergreen_mc_wait_for_idle(rdev)) {
2919                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2920         }
2921         evergreen_mc_resume(rdev, &save);
2922         /* we need to own VRAM, so turn off the VGA renderer here
2923          * to stop it overwriting our objects */
2924         rv515_vga_render_disable(rdev);
2925 }
2926
2927 /*
2928  * CP.
2929  */
2930 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2931 {
2932         struct radeon_ring *ring = &rdev->ring[ib->ring];
2933         u32 next_rptr;
2934
2935         /* set to DX10/11 mode */
2936         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2937         radeon_ring_write(ring, 1);
2938
2939         if (ring->rptr_save_reg) {
2940                 next_rptr = ring->wptr + 3 + 4;
2941                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2942                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2943                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2944                 radeon_ring_write(ring, next_rptr);
2945         } else if (rdev->wb.enabled) {
2946                 next_rptr = ring->wptr + 5 + 4;
2947                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2948                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2949                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2950                 radeon_ring_write(ring, next_rptr);
2951                 radeon_ring_write(ring, 0);
2952         }
2953
2954         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2955         radeon_ring_write(ring,
2956 #ifdef __BIG_ENDIAN
2957                           (2 << 0) |
2958 #endif
2959                           (ib->gpu_addr & 0xFFFFFFFC));
2960         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2961         radeon_ring_write(ring, ib->length_dw);
2962 }
2963
2964
2965 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2966 {
2967         const __be32 *fw_data;
2968         int i;
2969
2970         if (!rdev->me_fw || !rdev->pfp_fw)
2971                 return -EINVAL;
2972
2973         r700_cp_stop(rdev);
2974         WREG32(CP_RB_CNTL,
2975 #ifdef __BIG_ENDIAN
2976                BUF_SWAP_32BIT |
2977 #endif
2978                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2979
2980         fw_data = (const __be32 *)rdev->pfp_fw->data;
2981         WREG32(CP_PFP_UCODE_ADDR, 0);
2982         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2983                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2984         WREG32(CP_PFP_UCODE_ADDR, 0);
2985
2986         fw_data = (const __be32 *)rdev->me_fw->data;
2987         WREG32(CP_ME_RAM_WADDR, 0);
2988         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2989                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2990
2991         WREG32(CP_PFP_UCODE_ADDR, 0);
2992         WREG32(CP_ME_RAM_WADDR, 0);
2993         WREG32(CP_ME_RAM_RADDR, 0);
2994         return 0;
2995 }
2996
2997 static int evergreen_cp_start(struct radeon_device *rdev)
2998 {
2999         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3000         int r, i;
3001         uint32_t cp_me;
3002
3003         r = radeon_ring_lock(rdev, ring, 7);
3004         if (r) {
3005                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3006                 return r;
3007         }
3008         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3009         radeon_ring_write(ring, 0x1);
3010         radeon_ring_write(ring, 0x0);
3011         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3012         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3013         radeon_ring_write(ring, 0);
3014         radeon_ring_write(ring, 0);
3015         radeon_ring_unlock_commit(rdev, ring, false);
3016
3017         cp_me = 0xff;
3018         WREG32(CP_ME_CNTL, cp_me);
3019
3020         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3021         if (r) {
3022                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3023                 return r;
3024         }
3025
3026         /* setup clear context state */
3027         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3028         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3029
3030         for (i = 0; i < evergreen_default_size; i++)
3031                 radeon_ring_write(ring, evergreen_default_state[i]);
3032
3033         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3034         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3035
3036         /* set clear context state */
3037         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3038         radeon_ring_write(ring, 0);
3039
3040         /* SQ_VTX_BASE_VTX_LOC */
3041         radeon_ring_write(ring, 0xc0026f00);
3042         radeon_ring_write(ring, 0x00000000);
3043         radeon_ring_write(ring, 0x00000000);
3044         radeon_ring_write(ring, 0x00000000);
3045
3046         /* Clear consts */
3047         radeon_ring_write(ring, 0xc0036f00);
3048         radeon_ring_write(ring, 0x00000bc4);
3049         radeon_ring_write(ring, 0xffffffff);
3050         radeon_ring_write(ring, 0xffffffff);
3051         radeon_ring_write(ring, 0xffffffff);
3052
3053         radeon_ring_write(ring, 0xc0026900);
3054         radeon_ring_write(ring, 0x00000316);
3055         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3056         radeon_ring_write(ring, 0x00000010); /*  */
3057
3058         radeon_ring_unlock_commit(rdev, ring, false);
3059
3060         return 0;
3061 }
3062
3063 static int evergreen_cp_resume(struct radeon_device *rdev)
3064 {
3065         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3066         u32 tmp;
3067         u32 rb_bufsz;
3068         int r;
3069
3070         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3071         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3072                                  SOFT_RESET_PA |
3073                                  SOFT_RESET_SH |
3074                                  SOFT_RESET_VGT |
3075                                  SOFT_RESET_SPI |
3076                                  SOFT_RESET_SX));
3077         RREG32(GRBM_SOFT_RESET);
3078         mdelay(15);
3079         WREG32(GRBM_SOFT_RESET, 0);
3080         RREG32(GRBM_SOFT_RESET);
3081
3082         /* Set ring buffer size */
3083         rb_bufsz = order_base_2(ring->ring_size / 8);
3084         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3085 #ifdef __BIG_ENDIAN
3086         tmp |= BUF_SWAP_32BIT;
3087 #endif
3088         WREG32(CP_RB_CNTL, tmp);
3089         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3090         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3091
3092         /* Set the write pointer delay */
3093         WREG32(CP_RB_WPTR_DELAY, 0);
3094
3095         /* Initialize the ring buffer's read and write pointers */
3096         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3097         WREG32(CP_RB_RPTR_WR, 0);
3098         ring->wptr = 0;
3099         WREG32(CP_RB_WPTR, ring->wptr);
3100
3101         /* set the wb address whether it's enabled or not */
3102         WREG32(CP_RB_RPTR_ADDR,
3103                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3104         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3105         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3106
3107         if (rdev->wb.enabled)
3108                 WREG32(SCRATCH_UMSK, 0xff);
3109         else {
3110                 tmp |= RB_NO_UPDATE;
3111                 WREG32(SCRATCH_UMSK, 0);
3112         }
3113
3114         mdelay(1);
3115         WREG32(CP_RB_CNTL, tmp);
3116
3117         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3118         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3119
3120         evergreen_cp_start(rdev);
3121         ring->ready = true;
3122         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3123         if (r) {
3124                 ring->ready = false;
3125                 return r;
3126         }
3127         return 0;
3128 }
3129
3130 /*
3131  * Core functions
3132  */
3133 static void evergreen_gpu_init(struct radeon_device *rdev)
3134 {
3135         u32 gb_addr_config;
3136         u32 mc_shared_chmap, mc_arb_ramcfg;
3137         u32 sx_debug_1;
3138         u32 smx_dc_ctl0;
3139         u32 sq_config;
3140         u32 sq_lds_resource_mgmt;
3141         u32 sq_gpr_resource_mgmt_1;
3142         u32 sq_gpr_resource_mgmt_2;
3143         u32 sq_gpr_resource_mgmt_3;
3144         u32 sq_thread_resource_mgmt;
3145         u32 sq_thread_resource_mgmt_2;
3146         u32 sq_stack_resource_mgmt_1;
3147         u32 sq_stack_resource_mgmt_2;
3148         u32 sq_stack_resource_mgmt_3;
3149         u32 vgt_cache_invalidation;
3150         u32 hdp_host_path_cntl, tmp;
3151         u32 disabled_rb_mask;
3152         int i, j, ps_thread_count;
3153
3154         switch (rdev->family) {
3155         case CHIP_CYPRESS:
3156         case CHIP_HEMLOCK:
3157                 rdev->config.evergreen.num_ses = 2;
3158                 rdev->config.evergreen.max_pipes = 4;
3159                 rdev->config.evergreen.max_tile_pipes = 8;
3160                 rdev->config.evergreen.max_simds = 10;
3161                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3162                 rdev->config.evergreen.max_gprs = 256;
3163                 rdev->config.evergreen.max_threads = 248;
3164                 rdev->config.evergreen.max_gs_threads = 32;
3165                 rdev->config.evergreen.max_stack_entries = 512;
3166                 rdev->config.evergreen.sx_num_of_sets = 4;
3167                 rdev->config.evergreen.sx_max_export_size = 256;
3168                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3169                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3170                 rdev->config.evergreen.max_hw_contexts = 8;
3171                 rdev->config.evergreen.sq_num_cf_insts = 2;
3172
3173                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3174                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3175                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3176                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3177                 break;
3178         case CHIP_JUNIPER:
3179                 rdev->config.evergreen.num_ses = 1;
3180                 rdev->config.evergreen.max_pipes = 4;
3181                 rdev->config.evergreen.max_tile_pipes = 4;
3182                 rdev->config.evergreen.max_simds = 10;
3183                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3184                 rdev->config.evergreen.max_gprs = 256;
3185                 rdev->config.evergreen.max_threads = 248;
3186                 rdev->config.evergreen.max_gs_threads = 32;
3187                 rdev->config.evergreen.max_stack_entries = 512;
3188                 rdev->config.evergreen.sx_num_of_sets = 4;
3189                 rdev->config.evergreen.sx_max_export_size = 256;
3190                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3191                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3192                 rdev->config.evergreen.max_hw_contexts = 8;
3193                 rdev->config.evergreen.sq_num_cf_insts = 2;
3194
3195                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3196                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3197                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3198                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3199                 break;
3200         case CHIP_REDWOOD:
3201                 rdev->config.evergreen.num_ses = 1;
3202                 rdev->config.evergreen.max_pipes = 4;
3203                 rdev->config.evergreen.max_tile_pipes = 4;
3204                 rdev->config.evergreen.max_simds = 5;
3205                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3206                 rdev->config.evergreen.max_gprs = 256;
3207                 rdev->config.evergreen.max_threads = 248;
3208                 rdev->config.evergreen.max_gs_threads = 32;
3209                 rdev->config.evergreen.max_stack_entries = 256;
3210                 rdev->config.evergreen.sx_num_of_sets = 4;
3211                 rdev->config.evergreen.sx_max_export_size = 256;
3212                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3213                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3214                 rdev->config.evergreen.max_hw_contexts = 8;
3215                 rdev->config.evergreen.sq_num_cf_insts = 2;
3216
3217                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3218                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3219                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3220                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3221                 break;
3222         case CHIP_CEDAR:
3223         default:
3224                 rdev->config.evergreen.num_ses = 1;
3225                 rdev->config.evergreen.max_pipes = 2;
3226                 rdev->config.evergreen.max_tile_pipes = 2;
3227                 rdev->config.evergreen.max_simds = 2;
3228                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3229                 rdev->config.evergreen.max_gprs = 256;
3230                 rdev->config.evergreen.max_threads = 192;
3231                 rdev->config.evergreen.max_gs_threads = 16;
3232                 rdev->config.evergreen.max_stack_entries = 256;
3233                 rdev->config.evergreen.sx_num_of_sets = 4;
3234                 rdev->config.evergreen.sx_max_export_size = 128;
3235                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3236                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3237                 rdev->config.evergreen.max_hw_contexts = 4;
3238                 rdev->config.evergreen.sq_num_cf_insts = 1;
3239
3240                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3241                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3242                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3243                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3244                 break;
3245         case CHIP_PALM:
3246                 rdev->config.evergreen.num_ses = 1;
3247                 rdev->config.evergreen.max_pipes = 2;
3248                 rdev->config.evergreen.max_tile_pipes = 2;
3249                 rdev->config.evergreen.max_simds = 2;
3250                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3251                 rdev->config.evergreen.max_gprs = 256;
3252                 rdev->config.evergreen.max_threads = 192;
3253                 rdev->config.evergreen.max_gs_threads = 16;
3254                 rdev->config.evergreen.max_stack_entries = 256;
3255                 rdev->config.evergreen.sx_num_of_sets = 4;
3256                 rdev->config.evergreen.sx_max_export_size = 128;
3257                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3258                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3259                 rdev->config.evergreen.max_hw_contexts = 4;
3260                 rdev->config.evergreen.sq_num_cf_insts = 1;
3261
3262                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3263                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3264                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3265                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3266                 break;
3267         case CHIP_SUMO:
3268                 rdev->config.evergreen.num_ses = 1;
3269                 rdev->config.evergreen.max_pipes = 4;
3270                 rdev->config.evergreen.max_tile_pipes = 4;
3271                 if (rdev->pdev->device == 0x9648)
3272                         rdev->config.evergreen.max_simds = 3;
3273                 else if ((rdev->pdev->device == 0x9647) ||
3274                          (rdev->pdev->device == 0x964a))
3275                         rdev->config.evergreen.max_simds = 4;
3276                 else
3277                         rdev->config.evergreen.max_simds = 5;
3278                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3279                 rdev->config.evergreen.max_gprs = 256;
3280                 rdev->config.evergreen.max_threads = 248;
3281                 rdev->config.evergreen.max_gs_threads = 32;
3282                 rdev->config.evergreen.max_stack_entries = 256;
3283                 rdev->config.evergreen.sx_num_of_sets = 4;
3284                 rdev->config.evergreen.sx_max_export_size = 256;
3285                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3286                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3287                 rdev->config.evergreen.max_hw_contexts = 8;
3288                 rdev->config.evergreen.sq_num_cf_insts = 2;
3289
3290                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3291                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3292                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3293                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3294                 break;
3295         case CHIP_SUMO2:
3296                 rdev->config.evergreen.num_ses = 1;
3297                 rdev->config.evergreen.max_pipes = 4;
3298                 rdev->config.evergreen.max_tile_pipes = 4;
3299                 rdev->config.evergreen.max_simds = 2;
3300                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3301                 rdev->config.evergreen.max_gprs = 256;
3302                 rdev->config.evergreen.max_threads = 248;
3303                 rdev->config.evergreen.max_gs_threads = 32;
3304                 rdev->config.evergreen.max_stack_entries = 512;
3305                 rdev->config.evergreen.sx_num_of_sets = 4;
3306                 rdev->config.evergreen.sx_max_export_size = 256;
3307                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3308                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3309                 rdev->config.evergreen.max_hw_contexts = 4;
3310                 rdev->config.evergreen.sq_num_cf_insts = 2;
3311
3312                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3313                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3314                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3315                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3316                 break;
3317         case CHIP_BARTS:
3318                 rdev->config.evergreen.num_ses = 2;
3319                 rdev->config.evergreen.max_pipes = 4;
3320                 rdev->config.evergreen.max_tile_pipes = 8;
3321                 rdev->config.evergreen.max_simds = 7;
3322                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3323                 rdev->config.evergreen.max_gprs = 256;
3324                 rdev->config.evergreen.max_threads = 248;
3325                 rdev->config.evergreen.max_gs_threads = 32;
3326                 rdev->config.evergreen.max_stack_entries = 512;
3327                 rdev->config.evergreen.sx_num_of_sets = 4;
3328                 rdev->config.evergreen.sx_max_export_size = 256;
3329                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3330                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3331                 rdev->config.evergreen.max_hw_contexts = 8;
3332                 rdev->config.evergreen.sq_num_cf_insts = 2;
3333
3334                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3335                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3336                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3337                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3338                 break;
3339         case CHIP_TURKS:
3340                 rdev->config.evergreen.num_ses = 1;
3341                 rdev->config.evergreen.max_pipes = 4;
3342                 rdev->config.evergreen.max_tile_pipes = 4;
3343                 rdev->config.evergreen.max_simds = 6;
3344                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3345                 rdev->config.evergreen.max_gprs = 256;
3346                 rdev->config.evergreen.max_threads = 248;
3347                 rdev->config.evergreen.max_gs_threads = 32;
3348                 rdev->config.evergreen.max_stack_entries = 256;
3349                 rdev->config.evergreen.sx_num_of_sets = 4;
3350                 rdev->config.evergreen.sx_max_export_size = 256;
3351                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3352                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3353                 rdev->config.evergreen.max_hw_contexts = 8;
3354                 rdev->config.evergreen.sq_num_cf_insts = 2;
3355
3356                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3357                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3358                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3359                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3360                 break;
3361         case CHIP_CAICOS:
3362                 rdev->config.evergreen.num_ses = 1;
3363                 rdev->config.evergreen.max_pipes = 2;
3364                 rdev->config.evergreen.max_tile_pipes = 2;
3365                 rdev->config.evergreen.max_simds = 2;
3366                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3367                 rdev->config.evergreen.max_gprs = 256;
3368                 rdev->config.evergreen.max_threads = 192;
3369                 rdev->config.evergreen.max_gs_threads = 16;
3370                 rdev->config.evergreen.max_stack_entries = 256;
3371                 rdev->config.evergreen.sx_num_of_sets = 4;
3372                 rdev->config.evergreen.sx_max_export_size = 128;
3373                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3374                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3375                 rdev->config.evergreen.max_hw_contexts = 4;
3376                 rdev->config.evergreen.sq_num_cf_insts = 1;
3377
3378                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3379                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3380                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3381                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3382                 break;
3383         }
3384
3385         /* Initialize HDP */
3386         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3387                 WREG32((0x2c14 + j), 0x00000000);
3388                 WREG32((0x2c18 + j), 0x00000000);
3389                 WREG32((0x2c1c + j), 0x00000000);
3390                 WREG32((0x2c20 + j), 0x00000000);
3391                 WREG32((0x2c24 + j), 0x00000000);
3392         }
3393
3394         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3395         WREG32(SRBM_INT_CNTL, 0x1);
3396         WREG32(SRBM_INT_ACK, 0x1);
3397
3398         evergreen_fix_pci_max_read_req_size(rdev);
3399
3400         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3401         if ((rdev->family == CHIP_PALM) ||
3402             (rdev->family == CHIP_SUMO) ||
3403             (rdev->family == CHIP_SUMO2))
3404                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3405         else
3406                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3407
3408         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3409          * not have bank info, so create a custom tiling dword.
3410          * bits 3:0   num_pipes
3411          * bits 7:4   num_banks
3412          * bits 11:8  group_size
3413          * bits 15:12 row_size
3414          */
3415         rdev->config.evergreen.tile_config = 0;
3416         switch (rdev->config.evergreen.max_tile_pipes) {
3417         case 1:
3418         default:
3419                 rdev->config.evergreen.tile_config |= (0 << 0);
3420                 break;
3421         case 2:
3422                 rdev->config.evergreen.tile_config |= (1 << 0);
3423                 break;
3424         case 4:
3425                 rdev->config.evergreen.tile_config |= (2 << 0);
3426                 break;
3427         case 8:
3428                 rdev->config.evergreen.tile_config |= (3 << 0);
3429                 break;
3430         }
3431         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3432         if (rdev->flags & RADEON_IS_IGP)
3433                 rdev->config.evergreen.tile_config |= 1 << 4;
3434         else {
3435                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3436                 case 0: /* four banks */
3437                         rdev->config.evergreen.tile_config |= 0 << 4;
3438                         break;
3439                 case 1: /* eight banks */
3440                         rdev->config.evergreen.tile_config |= 1 << 4;
3441                         break;
3442                 case 2: /* sixteen banks */
3443                 default:
3444                         rdev->config.evergreen.tile_config |= 2 << 4;
3445                         break;
3446                 }
3447         }
3448         rdev->config.evergreen.tile_config |= 0 << 8;
3449         rdev->config.evergreen.tile_config |=
3450                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3451
3452         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3453                 u32 efuse_straps_4;
3454                 u32 efuse_straps_3;
3455
3456                 efuse_straps_4 = RREG32_RCU(0x204);
3457                 efuse_straps_3 = RREG32_RCU(0x203);
3458                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3459                       ((efuse_straps_3 & 0xf0000000) >> 28));
3460         } else {
3461                 tmp = 0;
3462                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3463                         u32 rb_disable_bitmap;
3464
3465                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3466                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3467                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3468                         tmp <<= 4;
3469                         tmp |= rb_disable_bitmap;
3470                 }
3471         }
3472         /* enabled rb are just the one not disabled :) */
3473         disabled_rb_mask = tmp;
3474         tmp = 0;
3475         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3476                 tmp |= (1 << i);
3477         /* if all the backends are disabled, fix it up here */
3478         if ((disabled_rb_mask & tmp) == tmp) {
3479                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3480                         disabled_rb_mask &= ~(1 << i);
3481         }
3482
3483         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3484                 u32 simd_disable_bitmap;
3485
3486                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3487                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3488                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3489                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3490                 tmp <<= 16;
3491                 tmp |= simd_disable_bitmap;
3492         }
3493         rdev->config.evergreen.active_simds = hweight32(~tmp);
3494
3495         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3496         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3497
3498         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3499         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3500         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3501         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3502         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3503         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3504         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3505
3506         if ((rdev->config.evergreen.max_backends == 1) &&
3507             (rdev->flags & RADEON_IS_IGP)) {
3508                 if ((disabled_rb_mask & 3) == 1) {
3509                         /* RB0 disabled, RB1 enabled */
3510                         tmp = 0x11111111;
3511                 } else {
3512                         /* RB1 disabled, RB0 enabled */
3513                         tmp = 0x00000000;
3514                 }
3515         } else {
3516                 tmp = gb_addr_config & NUM_PIPES_MASK;
3517                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3518                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3519         }
3520         rdev->config.evergreen.backend_map = tmp;
3521         WREG32(GB_BACKEND_MAP, tmp);
3522
3523         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3524         WREG32(CGTS_TCC_DISABLE, 0);
3525         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3526         WREG32(CGTS_USER_TCC_DISABLE, 0);
3527
3528         /* set HW defaults for 3D engine */
3529         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3530                                      ROQ_IB2_START(0x2b)));
3531
3532         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3533
3534         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3535                              SYNC_GRADIENT |
3536                              SYNC_WALKER |
3537                              SYNC_ALIGNER));
3538
3539         sx_debug_1 = RREG32(SX_DEBUG_1);
3540         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3541         WREG32(SX_DEBUG_1, sx_debug_1);
3542
3543
3544         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3545         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3546         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3547         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3548
3549         if (rdev->family <= CHIP_SUMO2)
3550                 WREG32(SMX_SAR_CTL0, 0x00010000);
3551
3552         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3553                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3554                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3555
3556         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3557                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3558                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3559
3560         WREG32(VGT_NUM_INSTANCES, 1);
3561         WREG32(SPI_CONFIG_CNTL, 0);
3562         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3563         WREG32(CP_PERFMON_CNTL, 0);
3564
3565         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3566                                   FETCH_FIFO_HIWATER(0x4) |
3567                                   DONE_FIFO_HIWATER(0xe0) |
3568                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3569
3570         sq_config = RREG32(SQ_CONFIG);
3571         sq_config &= ~(PS_PRIO(3) |
3572                        VS_PRIO(3) |
3573                        GS_PRIO(3) |
3574                        ES_PRIO(3));
3575         sq_config |= (VC_ENABLE |
3576                       EXPORT_SRC_C |
3577                       PS_PRIO(0) |
3578                       VS_PRIO(1) |
3579                       GS_PRIO(2) |
3580                       ES_PRIO(3));
3581
3582         switch (rdev->family) {
3583         case CHIP_CEDAR:
3584         case CHIP_PALM:
3585         case CHIP_SUMO:
3586         case CHIP_SUMO2:
3587         case CHIP_CAICOS:
3588                 /* no vertex cache */
3589                 sq_config &= ~VC_ENABLE;
3590                 break;
3591         default:
3592                 break;
3593         }
3594
3595         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3596
3597         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3598         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3599         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3600         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3601         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3602         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3603         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3604
3605         switch (rdev->family) {
3606         case CHIP_CEDAR:
3607         case CHIP_PALM:
3608         case CHIP_SUMO:
3609         case CHIP_SUMO2:
3610                 ps_thread_count = 96;
3611                 break;
3612         default:
3613                 ps_thread_count = 128;
3614                 break;
3615         }
3616
3617         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3618         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623
3624         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630
3631         WREG32(SQ_CONFIG, sq_config);
3632         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3633         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3634         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3635         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3636         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3637         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3638         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3639         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3640         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3641         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3642
3643         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3644                                           FORCE_EOV_MAX_REZ_CNT(255)));
3645
3646         switch (rdev->family) {
3647         case CHIP_CEDAR:
3648         case CHIP_PALM:
3649         case CHIP_SUMO:
3650         case CHIP_SUMO2:
3651         case CHIP_CAICOS:
3652                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3653                 break;
3654         default:
3655                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3656                 break;
3657         }
3658         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3659         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3660
3661         WREG32(VGT_GS_VERTEX_REUSE, 16);
3662         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3663         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3664
3665         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3666         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3667
3668         WREG32(CB_PERF_CTR0_SEL_0, 0);
3669         WREG32(CB_PERF_CTR0_SEL_1, 0);
3670         WREG32(CB_PERF_CTR1_SEL_0, 0);
3671         WREG32(CB_PERF_CTR1_SEL_1, 0);
3672         WREG32(CB_PERF_CTR2_SEL_0, 0);
3673         WREG32(CB_PERF_CTR2_SEL_1, 0);
3674         WREG32(CB_PERF_CTR3_SEL_0, 0);
3675         WREG32(CB_PERF_CTR3_SEL_1, 0);
3676
3677         /* clear render buffer base addresses */
3678         WREG32(CB_COLOR0_BASE, 0);
3679         WREG32(CB_COLOR1_BASE, 0);
3680         WREG32(CB_COLOR2_BASE, 0);
3681         WREG32(CB_COLOR3_BASE, 0);
3682         WREG32(CB_COLOR4_BASE, 0);
3683         WREG32(CB_COLOR5_BASE, 0);
3684         WREG32(CB_COLOR6_BASE, 0);
3685         WREG32(CB_COLOR7_BASE, 0);
3686         WREG32(CB_COLOR8_BASE, 0);
3687         WREG32(CB_COLOR9_BASE, 0);
3688         WREG32(CB_COLOR10_BASE, 0);
3689         WREG32(CB_COLOR11_BASE, 0);
3690
3691         /* set the shader const cache sizes to 0 */
3692         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3693                 WREG32(i, 0);
3694         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3695                 WREG32(i, 0);
3696
3697         tmp = RREG32(HDP_MISC_CNTL);
3698         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3699         WREG32(HDP_MISC_CNTL, tmp);
3700
3701         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3702         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3703
3704         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3705
3706         udelay(50);
3707
3708 }
3709
3710 int evergreen_mc_init(struct radeon_device *rdev)
3711 {
3712         u32 tmp;
3713         int chansize, numchan;
3714
3715         /* Get VRAM informations */
3716         rdev->mc.vram_is_ddr = true;
3717         if ((rdev->family == CHIP_PALM) ||
3718             (rdev->family == CHIP_SUMO) ||
3719             (rdev->family == CHIP_SUMO2))
3720                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3721         else
3722                 tmp = RREG32(MC_ARB_RAMCFG);
3723         if (tmp & CHANSIZE_OVERRIDE) {
3724                 chansize = 16;
3725         } else if (tmp & CHANSIZE_MASK) {
3726                 chansize = 64;
3727         } else {
3728                 chansize = 32;
3729         }
3730         tmp = RREG32(MC_SHARED_CHMAP);
3731         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3732         case 0:
3733         default:
3734                 numchan = 1;
3735                 break;
3736         case 1:
3737                 numchan = 2;
3738                 break;
3739         case 2:
3740                 numchan = 4;
3741                 break;
3742         case 3:
3743                 numchan = 8;
3744                 break;
3745         }
3746         rdev->mc.vram_width = numchan * chansize;
3747         /* Could aper size report 0 ? */
3748         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3749         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3750         /* Setup GPU memory space */
3751         if ((rdev->family == CHIP_PALM) ||
3752             (rdev->family == CHIP_SUMO) ||
3753             (rdev->family == CHIP_SUMO2)) {
3754                 /* size in bytes on fusion */
3755                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3756                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3757         } else {
3758                 /* size in MB on evergreen/cayman/tn */
3759                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3760                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3761         }
3762         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3763         r700_vram_gtt_location(rdev, &rdev->mc);
3764         radeon_update_bandwidth_info(rdev);
3765
3766         return 0;
3767 }
3768
3769 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3770 {
3771         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3772                 RREG32(GRBM_STATUS));
3773         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3774                 RREG32(GRBM_STATUS_SE0));
3775         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3776                 RREG32(GRBM_STATUS_SE1));
3777         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3778                 RREG32(SRBM_STATUS));
3779         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3780                 RREG32(SRBM_STATUS2));
3781         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3782                 RREG32(CP_STALLED_STAT1));
3783         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3784                 RREG32(CP_STALLED_STAT2));
3785         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3786                 RREG32(CP_BUSY_STAT));
3787         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3788                 RREG32(CP_STAT));
3789         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3790                 RREG32(DMA_STATUS_REG));
3791         if (rdev->family >= CHIP_CAYMAN) {
3792                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3793                          RREG32(DMA_STATUS_REG + 0x800));
3794         }
3795 }
3796
3797 bool evergreen_is_display_hung(struct radeon_device *rdev)
3798 {
3799         u32 crtc_hung = 0;
3800         u32 crtc_status[6];
3801         u32 i, j, tmp;
3802
3803         for (i = 0; i < rdev->num_crtc; i++) {
3804                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3805                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3806                         crtc_hung |= (1 << i);
3807                 }
3808         }
3809
3810         for (j = 0; j < 10; j++) {
3811                 for (i = 0; i < rdev->num_crtc; i++) {
3812                         if (crtc_hung & (1 << i)) {
3813                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3814                                 if (tmp != crtc_status[i])
3815                                         crtc_hung &= ~(1 << i);
3816                         }
3817                 }
3818                 if (crtc_hung == 0)
3819                         return false;
3820                 udelay(100);
3821         }
3822
3823         return true;
3824 }
3825
3826 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3827 {
3828         u32 reset_mask = 0;
3829         u32 tmp;
3830
3831         /* GRBM_STATUS */
3832         tmp = RREG32(GRBM_STATUS);
3833         if (tmp & (PA_BUSY | SC_BUSY |
3834                    SH_BUSY | SX_BUSY |
3835                    TA_BUSY | VGT_BUSY |
3836                    DB_BUSY | CB_BUSY |
3837                    SPI_BUSY | VGT_BUSY_NO_DMA))
3838                 reset_mask |= RADEON_RESET_GFX;
3839
3840         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3841                    CP_BUSY | CP_COHERENCY_BUSY))
3842                 reset_mask |= RADEON_RESET_CP;
3843
3844         if (tmp & GRBM_EE_BUSY)
3845                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3846
3847         /* DMA_STATUS_REG */
3848         tmp = RREG32(DMA_STATUS_REG);
3849         if (!(tmp & DMA_IDLE))
3850                 reset_mask |= RADEON_RESET_DMA;
3851
3852         /* SRBM_STATUS2 */
3853         tmp = RREG32(SRBM_STATUS2);
3854         if (tmp & DMA_BUSY)
3855                 reset_mask |= RADEON_RESET_DMA;
3856
3857         /* SRBM_STATUS */
3858         tmp = RREG32(SRBM_STATUS);
3859         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3860                 reset_mask |= RADEON_RESET_RLC;
3861
3862         if (tmp & IH_BUSY)
3863                 reset_mask |= RADEON_RESET_IH;
3864
3865         if (tmp & SEM_BUSY)
3866                 reset_mask |= RADEON_RESET_SEM;
3867
3868         if (tmp & GRBM_RQ_PENDING)
3869                 reset_mask |= RADEON_RESET_GRBM;
3870
3871         if (tmp & VMC_BUSY)
3872                 reset_mask |= RADEON_RESET_VMC;
3873
3874         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3875                    MCC_BUSY | MCD_BUSY))
3876                 reset_mask |= RADEON_RESET_MC;
3877
3878         if (evergreen_is_display_hung(rdev))
3879                 reset_mask |= RADEON_RESET_DISPLAY;
3880
3881         /* VM_L2_STATUS */
3882         tmp = RREG32(VM_L2_STATUS);
3883         if (tmp & L2_BUSY)
3884                 reset_mask |= RADEON_RESET_VMC;
3885
3886         /* Skip MC reset as it's mostly likely not hung, just busy */
3887         if (reset_mask & RADEON_RESET_MC) {
3888                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3889                 reset_mask &= ~RADEON_RESET_MC;
3890         }
3891
3892         return reset_mask;
3893 }
3894
3895 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3896 {
3897         struct evergreen_mc_save save;
3898         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3899         u32 tmp;
3900
3901         if (reset_mask == 0)
3902                 return;
3903
3904         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3905
3906         evergreen_print_gpu_status_regs(rdev);
3907
3908         /* Disable CP parsing/prefetching */
3909         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3910
3911         if (reset_mask & RADEON_RESET_DMA) {
3912                 /* Disable DMA */
3913                 tmp = RREG32(DMA_RB_CNTL);
3914                 tmp &= ~DMA_RB_ENABLE;
3915                 WREG32(DMA_RB_CNTL, tmp);
3916         }
3917
3918         udelay(50);
3919
3920         evergreen_mc_stop(rdev, &save);
3921         if (evergreen_mc_wait_for_idle(rdev)) {
3922                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3923         }
3924
3925         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3926                 grbm_soft_reset |= SOFT_RESET_DB |
3927                         SOFT_RESET_CB |
3928                         SOFT_RESET_PA |
3929                         SOFT_RESET_SC |
3930                         SOFT_RESET_SPI |
3931                         SOFT_RESET_SX |
3932                         SOFT_RESET_SH |
3933                         SOFT_RESET_TC |
3934                         SOFT_RESET_TA |
3935                         SOFT_RESET_VC |
3936                         SOFT_RESET_VGT;
3937         }
3938
3939         if (reset_mask & RADEON_RESET_CP) {
3940                 grbm_soft_reset |= SOFT_RESET_CP |
3941                         SOFT_RESET_VGT;
3942
3943                 srbm_soft_reset |= SOFT_RESET_GRBM;
3944         }
3945
3946         if (reset_mask & RADEON_RESET_DMA)
3947                 srbm_soft_reset |= SOFT_RESET_DMA;
3948
3949         if (reset_mask & RADEON_RESET_DISPLAY)
3950                 srbm_soft_reset |= SOFT_RESET_DC;
3951
3952         if (reset_mask & RADEON_RESET_RLC)
3953                 srbm_soft_reset |= SOFT_RESET_RLC;
3954
3955         if (reset_mask & RADEON_RESET_SEM)
3956                 srbm_soft_reset |= SOFT_RESET_SEM;
3957
3958         if (reset_mask & RADEON_RESET_IH)
3959                 srbm_soft_reset |= SOFT_RESET_IH;
3960
3961         if (reset_mask & RADEON_RESET_GRBM)
3962                 srbm_soft_reset |= SOFT_RESET_GRBM;
3963
3964         if (reset_mask & RADEON_RESET_VMC)
3965                 srbm_soft_reset |= SOFT_RESET_VMC;
3966
3967         if (!(rdev->flags & RADEON_IS_IGP)) {
3968                 if (reset_mask & RADEON_RESET_MC)
3969                         srbm_soft_reset |= SOFT_RESET_MC;
3970         }
3971
3972         if (grbm_soft_reset) {
3973                 tmp = RREG32(GRBM_SOFT_RESET);
3974                 tmp |= grbm_soft_reset;
3975                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3976                 WREG32(GRBM_SOFT_RESET, tmp);
3977                 tmp = RREG32(GRBM_SOFT_RESET);
3978
3979                 udelay(50);
3980
3981                 tmp &= ~grbm_soft_reset;
3982                 WREG32(GRBM_SOFT_RESET, tmp);
3983                 tmp = RREG32(GRBM_SOFT_RESET);
3984         }
3985
3986         if (srbm_soft_reset) {
3987                 tmp = RREG32(SRBM_SOFT_RESET);
3988                 tmp |= srbm_soft_reset;
3989                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3990                 WREG32(SRBM_SOFT_RESET, tmp);
3991                 tmp = RREG32(SRBM_SOFT_RESET);
3992
3993                 udelay(50);
3994
3995                 tmp &= ~srbm_soft_reset;
3996                 WREG32(SRBM_SOFT_RESET, tmp);
3997                 tmp = RREG32(SRBM_SOFT_RESET);
3998         }
3999
4000         /* Wait a little for things to settle down */
4001         udelay(50);
4002
4003         evergreen_mc_resume(rdev, &save);
4004         udelay(50);
4005
4006         evergreen_print_gpu_status_regs(rdev);
4007 }
4008
4009 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4010 {
4011         struct evergreen_mc_save save;
4012         u32 tmp, i;
4013
4014         dev_info(rdev->dev, "GPU pci config reset\n");
4015
4016         /* disable dpm? */
4017
4018         /* Disable CP parsing/prefetching */
4019         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4020         udelay(50);
4021         /* Disable DMA */
4022         tmp = RREG32(DMA_RB_CNTL);
4023         tmp &= ~DMA_RB_ENABLE;
4024         WREG32(DMA_RB_CNTL, tmp);
4025         /* XXX other engines? */
4026
4027         /* halt the rlc */
4028         r600_rlc_stop(rdev);
4029
4030         udelay(50);
4031
4032         /* set mclk/sclk to bypass */
4033         rv770_set_clk_bypass_mode(rdev);
4034         /* disable BM */
4035         pci_clear_master(rdev->pdev);
4036         /* disable mem access */
4037         evergreen_mc_stop(rdev, &save);
4038         if (evergreen_mc_wait_for_idle(rdev)) {
4039                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4040         }
4041         /* reset */
4042         radeon_pci_config_reset(rdev);
4043         /* wait for asic to come out of reset */
4044         for (i = 0; i < rdev->usec_timeout; i++) {
4045                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4046                         break;
4047                 udelay(1);
4048         }
4049 }
4050
4051 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4052 {
4053         u32 reset_mask;
4054
4055         if (hard) {
4056                 evergreen_gpu_pci_config_reset(rdev);
4057                 return 0;
4058         }
4059
4060         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4061
4062         if (reset_mask)
4063                 r600_set_bios_scratch_engine_hung(rdev, true);
4064
4065         /* try soft reset */
4066         evergreen_gpu_soft_reset(rdev, reset_mask);
4067
4068         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4069
4070         /* try pci config reset */
4071         if (reset_mask && radeon_hard_reset)
4072                 evergreen_gpu_pci_config_reset(rdev);
4073
4074         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4075
4076         if (!reset_mask)
4077                 r600_set_bios_scratch_engine_hung(rdev, false);
4078
4079         return 0;
4080 }
4081
4082 /**
4083  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4084  *
4085  * @rdev: radeon_device pointer
4086  * @ring: radeon_ring structure holding ring information
4087  *
4088  * Check if the GFX engine is locked up.
4089  * Returns true if the engine appears to be locked up, false if not.
4090  */
4091 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4092 {
4093         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4094
4095         if (!(reset_mask & (RADEON_RESET_GFX |
4096                             RADEON_RESET_COMPUTE |
4097                             RADEON_RESET_CP))) {
4098                 radeon_ring_lockup_update(rdev, ring);
4099                 return false;
4100         }
4101         return radeon_ring_test_lockup(rdev, ring);
4102 }
4103
4104 /*
4105  * RLC
4106  */
4107 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4108 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
4109
4110 void sumo_rlc_fini(struct radeon_device *rdev)
4111 {
4112         int r;
4113
4114         /* save restore block */
4115         if (rdev->rlc.save_restore_obj) {
4116                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4117                 if (unlikely(r != 0))
4118                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4119                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4120                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4121
4122                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4123                 rdev->rlc.save_restore_obj = NULL;
4124         }
4125
4126         /* clear state block */
4127         if (rdev->rlc.clear_state_obj) {
4128                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4129                 if (unlikely(r != 0))
4130                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4131                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4132                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4133
4134                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4135                 rdev->rlc.clear_state_obj = NULL;
4136         }
4137
4138         /* clear state block */
4139         if (rdev->rlc.cp_table_obj) {
4140                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4141                 if (unlikely(r != 0))
4142                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4143                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4144                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4145
4146                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4147                 rdev->rlc.cp_table_obj = NULL;
4148         }
4149 }
4150
4151 #define CP_ME_TABLE_SIZE    96
4152
4153 int sumo_rlc_init(struct radeon_device *rdev)
4154 {
4155         const u32 *src_ptr;
4156         volatile u32 *dst_ptr;
4157         u32 dws, data, i, j, k, reg_num;
4158         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4159         u64 reg_list_mc_addr;
4160         const struct cs_section_def *cs_data;
4161         int r;
4162
4163         src_ptr = rdev->rlc.reg_list;
4164         dws = rdev->rlc.reg_list_size;
4165         if (rdev->family >= CHIP_BONAIRE) {
4166                 dws += (5 * 16) + 48 + 48 + 64;
4167         }
4168         cs_data = rdev->rlc.cs_data;
4169
4170         if (src_ptr) {
4171                 /* save restore block */
4172                 if (rdev->rlc.save_restore_obj == NULL) {
4173                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4174                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4175                                              NULL, &rdev->rlc.save_restore_obj);
4176                         if (r) {
4177                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4178                                 return r;
4179                         }
4180                 }
4181
4182                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4183                 if (unlikely(r != 0)) {
4184                         sumo_rlc_fini(rdev);
4185                         return r;
4186                 }
4187                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4188                                   &rdev->rlc.save_restore_gpu_addr);
4189                 if (r) {
4190                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4191                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4192                         sumo_rlc_fini(rdev);
4193                         return r;
4194                 }
4195
4196                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4197                 if (r) {
4198                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4199                         sumo_rlc_fini(rdev);
4200                         return r;
4201                 }
4202                 /* write the sr buffer */
4203                 dst_ptr = rdev->rlc.sr_ptr;
4204                 if (rdev->family >= CHIP_TAHITI) {
4205                         /* SI */
4206                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4207                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4208                 } else {
4209                         /* ON/LN/TN */
4210                         /* format:
4211                          * dw0: (reg2 << 16) | reg1
4212                          * dw1: reg1 save space
4213                          * dw2: reg2 save space
4214                          */
4215                         for (i = 0; i < dws; i++) {
4216                                 data = src_ptr[i] >> 2;
4217                                 i++;
4218                                 if (i < dws)
4219                                         data |= (src_ptr[i] >> 2) << 16;
4220                                 j = (((i - 1) * 3) / 2);
4221                                 dst_ptr[j] = cpu_to_le32(data);
4222                         }
4223                         j = ((i * 3) / 2);
4224                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4225                 }
4226                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4227                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4228         }
4229
4230         if (cs_data) {
4231                 /* clear state block */
4232                 if (rdev->family >= CHIP_BONAIRE) {
4233                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4234                 } else if (rdev->family >= CHIP_TAHITI) {
4235                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4236                         dws = rdev->rlc.clear_state_size + (256 / 4);
4237                 } else {
4238                         reg_list_num = 0;
4239                         dws = 0;
4240                         for (i = 0; cs_data[i].section != NULL; i++) {
4241                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4242                                         reg_list_num++;
4243                                         dws += cs_data[i].section[j].reg_count;
4244                                 }
4245                         }
4246                         reg_list_blk_index = (3 * reg_list_num + 2);
4247                         dws += reg_list_blk_index;
4248                         rdev->rlc.clear_state_size = dws;
4249                 }
4250
4251                 if (rdev->rlc.clear_state_obj == NULL) {
4252                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4253                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4254                                              NULL, &rdev->rlc.clear_state_obj);
4255                         if (r) {
4256                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4257                                 sumo_rlc_fini(rdev);
4258                                 return r;
4259                         }
4260                 }
4261                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4262                 if (unlikely(r != 0)) {
4263                         sumo_rlc_fini(rdev);
4264                         return r;
4265                 }
4266                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4267                                   &rdev->rlc.clear_state_gpu_addr);
4268                 if (r) {
4269                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4270                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4271                         sumo_rlc_fini(rdev);
4272                         return r;
4273                 }
4274
4275                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4276                 if (r) {
4277                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4278                         sumo_rlc_fini(rdev);
4279                         return r;
4280                 }
4281                 /* set up the cs buffer */
4282                 dst_ptr = rdev->rlc.cs_ptr;
4283                 if (rdev->family >= CHIP_BONAIRE) {
4284                         cik_get_csb_buffer(rdev, dst_ptr);
4285                 } else if (rdev->family >= CHIP_TAHITI) {
4286                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4287                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4288                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4289                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4290                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4291                 } else {
4292                         reg_list_hdr_blk_index = 0;
4293                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4294                         data = upper_32_bits(reg_list_mc_addr);
4295                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4296                         reg_list_hdr_blk_index++;
4297                         for (i = 0; cs_data[i].section != NULL; i++) {
4298                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4299                                         reg_num = cs_data[i].section[j].reg_count;
4300                                         data = reg_list_mc_addr & 0xffffffff;
4301                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4302                                         reg_list_hdr_blk_index++;
4303
4304                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4305                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4306                                         reg_list_hdr_blk_index++;
4307
4308                                         data = 0x08000000 | (reg_num * 4);
4309                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4310                                         reg_list_hdr_blk_index++;
4311
4312                                         for (k = 0; k < reg_num; k++) {
4313                                                 data = cs_data[i].section[j].extent[k];
4314                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4315                                         }
4316                                         reg_list_mc_addr += reg_num * 4;
4317                                         reg_list_blk_index += reg_num;
4318                                 }
4319                         }
4320                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4321                 }
4322                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4323                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4324         }
4325
4326         if (rdev->rlc.cp_table_size) {
4327                 if (rdev->rlc.cp_table_obj == NULL) {
4328                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4329                                              PAGE_SIZE, true,
4330                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4331                                              NULL, &rdev->rlc.cp_table_obj);
4332                         if (r) {
4333                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4334                                 sumo_rlc_fini(rdev);
4335                                 return r;
4336                         }
4337                 }
4338
4339                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4340                 if (unlikely(r != 0)) {
4341                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4342                         sumo_rlc_fini(rdev);
4343                         return r;
4344                 }
4345                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4346                                   &rdev->rlc.cp_table_gpu_addr);
4347                 if (r) {
4348                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4349                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4350                         sumo_rlc_fini(rdev);
4351                         return r;
4352                 }
4353                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4354                 if (r) {
4355                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4356                         sumo_rlc_fini(rdev);
4357                         return r;
4358                 }
4359
4360                 cik_init_cp_pg_table(rdev);
4361
4362                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4363                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4364
4365         }
4366
4367         return 0;
4368 }
4369
4370 static void evergreen_rlc_start(struct radeon_device *rdev)
4371 {
4372         u32 mask = RLC_ENABLE;
4373
4374         if (rdev->flags & RADEON_IS_IGP) {
4375                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4376         }
4377
4378         WREG32(RLC_CNTL, mask);
4379 }
4380
4381 int evergreen_rlc_resume(struct radeon_device *rdev)
4382 {
4383         u32 i;
4384         const __be32 *fw_data;
4385
4386         if (!rdev->rlc_fw)
4387                 return -EINVAL;
4388
4389         r600_rlc_stop(rdev);
4390
4391         WREG32(RLC_HB_CNTL, 0);
4392
4393         if (rdev->flags & RADEON_IS_IGP) {
4394                 if (rdev->family == CHIP_ARUBA) {
4395                         u32 always_on_bitmap =
4396                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4397                         /* find out the number of active simds */
4398                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4399                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4400                         tmp = hweight32(~tmp);
4401                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4402                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4403                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4404                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4405                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4406                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4407                         }
4408                 } else {
4409                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4410                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4411                 }
4412                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4413                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4414         } else {
4415                 WREG32(RLC_HB_BASE, 0);
4416                 WREG32(RLC_HB_RPTR, 0);
4417                 WREG32(RLC_HB_WPTR, 0);
4418                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4419                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4420         }
4421         WREG32(RLC_MC_CNTL, 0);
4422         WREG32(RLC_UCODE_CNTL, 0);
4423
4424         fw_data = (const __be32 *)rdev->rlc_fw->data;
4425         if (rdev->family >= CHIP_ARUBA) {
4426                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4427                         WREG32(RLC_UCODE_ADDR, i);
4428                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4429                 }
4430         } else if (rdev->family >= CHIP_CAYMAN) {
4431                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4432                         WREG32(RLC_UCODE_ADDR, i);
4433                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4434                 }
4435         } else {
4436                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4437                         WREG32(RLC_UCODE_ADDR, i);
4438                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4439                 }
4440         }
4441         WREG32(RLC_UCODE_ADDR, 0);
4442
4443         evergreen_rlc_start(rdev);
4444
4445         return 0;
4446 }
4447
4448 /* Interrupts */
4449
4450 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4451 {
4452         if (crtc >= rdev->num_crtc)
4453                 return 0;
4454         else
4455                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4456 }
4457
4458 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4459 {
4460         int i;
4461         u32 tmp;
4462
4463         if (rdev->family >= CHIP_CAYMAN) {
4464                 cayman_cp_int_cntl_setup(rdev, 0,
4465                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4466                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4467                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4468                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4469                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4470         } else
4471                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4472         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4473         WREG32(DMA_CNTL, tmp);
4474         WREG32(GRBM_INT_CNTL, 0);
4475         WREG32(SRBM_INT_CNTL, 0);
4476         for (i = 0; i < rdev->num_crtc; i++)
4477                 WREG32(INT_MASK + crtc_offsets[i], 0);
4478         for (i = 0; i < rdev->num_crtc; i++)
4479                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4480
4481         /* only one DAC on DCE5 */
4482         if (!ASIC_IS_DCE5(rdev))
4483                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4484         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4485
4486         for (i = 0; i < 6; i++)
4487                 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4488 }
4489
4490 /* Note that the order we write back regs here is important */
4491 int evergreen_irq_set(struct radeon_device *rdev)
4492 {
4493         int i;
4494         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4495         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4496         u32 grbm_int_cntl = 0;
4497         u32 dma_cntl, dma_cntl1 = 0;
4498         u32 thermal_int = 0;
4499
4500         if (!rdev->irq.installed) {
4501                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4502                 return -EINVAL;
4503         }
4504         /* don't enable anything if the ih is disabled */
4505         if (!rdev->ih.enabled) {
4506                 r600_disable_interrupts(rdev);
4507                 /* force the active interrupt state to all disabled */
4508                 evergreen_disable_interrupt_state(rdev);
4509                 return 0;
4510         }
4511
4512         if (rdev->family == CHIP_ARUBA)
4513                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4514                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4515         else
4516                 thermal_int = RREG32(CG_THERMAL_INT) &
4517                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4518
4519         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4520
4521         if (rdev->family >= CHIP_CAYMAN) {
4522                 /* enable CP interrupts on all rings */
4523                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4524                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4525                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4526                 }
4527                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4528                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4529                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4530                 }
4531                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4532                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4533                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4534                 }
4535         } else {
4536                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4537                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4538                         cp_int_cntl |= RB_INT_ENABLE;
4539                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4540                 }
4541         }
4542
4543         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4544                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4545                 dma_cntl |= TRAP_ENABLE;
4546         }
4547
4548         if (rdev->family >= CHIP_CAYMAN) {
4549                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4550                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4551                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4552                         dma_cntl1 |= TRAP_ENABLE;
4553                 }
4554         }
4555
4556         if (rdev->irq.dpm_thermal) {
4557                 DRM_DEBUG("dpm thermal\n");
4558                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4559         }
4560
4561         if (rdev->family >= CHIP_CAYMAN) {
4562                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4563                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4564                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4565         } else
4566                 WREG32(CP_INT_CNTL, cp_int_cntl);
4567
4568         WREG32(DMA_CNTL, dma_cntl);
4569
4570         if (rdev->family >= CHIP_CAYMAN)
4571                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4572
4573         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4574
4575         for (i = 0; i < rdev->num_crtc; i++) {
4576                 radeon_irq_kms_set_irq_n_enabled(
4577                     rdev, INT_MASK + crtc_offsets[i],
4578                     VBLANK_INT_MASK,
4579                     rdev->irq.crtc_vblank_int[i] ||
4580                     atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4581         }
4582
4583         for (i = 0; i < rdev->num_crtc; i++)
4584                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4585
4586         for (i = 0; i < 6; i++) {
4587                 radeon_irq_kms_set_irq_n_enabled(
4588                     rdev, DC_HPDx_INT_CONTROL(i),
4589                     DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4590                     rdev->irq.hpd[i], "HPD", i);
4591         }
4592
4593         if (rdev->family == CHIP_ARUBA)
4594                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4595         else
4596                 WREG32(CG_THERMAL_INT, thermal_int);
4597
4598         for (i = 0; i < 6; i++) {
4599                 radeon_irq_kms_set_irq_n_enabled(
4600                     rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4601                     AFMT_AZ_FORMAT_WTRIG_MASK,
4602                     rdev->irq.afmt[i], "HDMI", i);
4603         }
4604
4605         /* posting read */
4606         RREG32(SRBM_STATUS);
4607
4608         return 0;
4609 }
4610
4611 /* Note that the order we write back regs here is important */
4612 static void evergreen_irq_ack(struct radeon_device *rdev)
4613 {
4614         int i, j;
4615         u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4616         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4617         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4618
4619         for (i = 0; i < 6; i++) {
4620                 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4621                 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4622                 if (i < rdev->num_crtc)
4623                         grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4624         }
4625
4626         /* We write back each interrupt register in pairs of two */
4627         for (i = 0; i < rdev->num_crtc; i += 2) {
4628                 for (j = i; j < (i + 2); j++) {
4629                         if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4630                                 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4631                                        GRPH_PFLIP_INT_CLEAR);
4632                 }
4633
4634                 for (j = i; j < (i + 2); j++) {
4635                         if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4636                                 WREG32(VBLANK_STATUS + crtc_offsets[j],
4637                                        VBLANK_ACK);
4638                         if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4639                                 WREG32(VLINE_STATUS + crtc_offsets[j],
4640                                        VLINE_ACK);
4641                 }
4642         }
4643
4644         for (i = 0; i < 6; i++) {
4645                 if (disp_int[i] & DC_HPD1_INTERRUPT)
4646                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4647         }
4648
4649         for (i = 0; i < 6; i++) {
4650                 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4651                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4652         }
4653
4654         for (i = 0; i < 6; i++) {
4655                 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4656                         WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4657                                   AFMT_AZ_FORMAT_WTRIG_ACK);
4658         }
4659 }
4660
4661 static void evergreen_irq_disable(struct radeon_device *rdev)
4662 {
4663         r600_disable_interrupts(rdev);
4664         /* Wait and acknowledge irq */
4665         mdelay(1);
4666         evergreen_irq_ack(rdev);
4667         evergreen_disable_interrupt_state(rdev);
4668 }
4669
4670 void evergreen_irq_suspend(struct radeon_device *rdev)
4671 {
4672         evergreen_irq_disable(rdev);
4673         r600_rlc_stop(rdev);
4674 }
4675
4676 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4677 {
4678         u32 wptr, tmp;
4679
4680         if (rdev->wb.enabled)
4681                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4682         else
4683                 wptr = RREG32(IH_RB_WPTR);
4684
4685         if (wptr & RB_OVERFLOW) {
4686                 wptr &= ~RB_OVERFLOW;
4687                 /* When a ring buffer overflow happen start parsing interrupt
4688                  * from the last not overwritten vector (wptr + 16). Hopefully
4689                  * this should allow us to catchup.
4690                  */
4691                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4692                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4693                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4694                 tmp = RREG32(IH_RB_CNTL);
4695                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4696                 WREG32(IH_RB_CNTL, tmp);
4697         }
4698         return (wptr & rdev->ih.ptr_mask);
4699 }
4700
4701 int evergreen_irq_process(struct radeon_device *rdev)
4702 {
4703         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4704         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4705         u32 crtc_idx, hpd_idx, afmt_idx;
4706         u32 mask;
4707         u32 wptr;
4708         u32 rptr;
4709         u32 src_id, src_data;
4710         u32 ring_index;
4711         bool queue_hotplug = false;
4712         bool queue_hdmi = false;
4713         bool queue_dp = false;
4714         bool queue_thermal = false;
4715         u32 status, addr;
4716         const char *event_name;
4717
4718         if (!rdev->ih.enabled || rdev->shutdown)
4719                 return IRQ_NONE;
4720
4721         wptr = evergreen_get_ih_wptr(rdev);
4722
4723 restart_ih:
4724         /* is somebody else already processing irqs? */
4725         if (atomic_xchg(&rdev->ih.lock, 1))
4726                 return IRQ_NONE;
4727
4728         rptr = rdev->ih.rptr;
4729         DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4730
4731         /* Order reading of wptr vs. reading of IH ring data */
4732         rmb();
4733
4734         /* display interrupts */
4735         evergreen_irq_ack(rdev);
4736
4737         while (rptr != wptr) {
4738                 /* wptr/rptr are in bytes! */
4739                 ring_index = rptr / 4;
4740                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4741                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4742
4743                 switch (src_id) {
4744                 case 1: /* D1 vblank/vline */
4745                 case 2: /* D2 vblank/vline */
4746                 case 3: /* D3 vblank/vline */
4747                 case 4: /* D4 vblank/vline */
4748                 case 5: /* D5 vblank/vline */
4749                 case 6: /* D6 vblank/vline */
4750                         crtc_idx = src_id - 1;
4751
4752                         if (src_data == 0) { /* vblank */
4753                                 mask = LB_D1_VBLANK_INTERRUPT;
4754                                 event_name = "vblank";
4755
4756                                 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4757                                         drm_handle_vblank(rdev->ddev, crtc_idx);
4758                                         rdev->pm.vblank_sync = true;
4759                                         wake_up(&rdev->irq.vblank_queue);
4760                                 }
4761                                 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4762                                         radeon_crtc_handle_vblank(rdev,
4763                                                                   crtc_idx);
4764                                 }
4765
4766                         } else if (src_data == 1) { /* vline */
4767                                 mask = LB_D1_VLINE_INTERRUPT;
4768                                 event_name = "vline";
4769                         } else {
4770                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4771                                           src_id, src_data);
4772                                 break;
4773                         }
4774
4775                         if (!(disp_int[crtc_idx] & mask)) {
4776                                 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4777                                           crtc_idx + 1, event_name);
4778                         }
4779
4780                         disp_int[crtc_idx] &= ~mask;
4781                         DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4782
4783                         break;
4784                 case 8: /* D1 page flip */
4785                 case 10: /* D2 page flip */
4786                 case 12: /* D3 page flip */
4787                 case 14: /* D4 page flip */
4788                 case 16: /* D5 page flip */
4789                 case 18: /* D6 page flip */
4790                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4791                         if (radeon_use_pflipirq > 0)
4792                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4793                         break;
4794                 case 42: /* HPD hotplug */
4795                         if (src_data <= 5) {
4796                                 hpd_idx = src_data;
4797                                 mask = DC_HPD1_INTERRUPT;
4798                                 queue_hotplug = true;
4799                                 event_name = "HPD";
4800
4801                         } else if (src_data <= 11) {
4802                                 hpd_idx = src_data - 6;
4803                                 mask = DC_HPD1_RX_INTERRUPT;
4804                                 queue_dp = true;
4805                                 event_name = "HPD_RX";
4806
4807                         } else {
4808                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4809                                           src_id, src_data);
4810                                 break;
4811                         }
4812
4813                         if (!(disp_int[hpd_idx] & mask))
4814                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4815
4816                         disp_int[hpd_idx] &= ~mask;
4817                         DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4818
4819                         break;
4820                 case 44: /* hdmi */
4821                         afmt_idx = src_data;
4822                         if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4823                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4824
4825                         if (afmt_idx > 5) {
4826                                 DRM_ERROR("Unhandled interrupt: %d %d\n",
4827                                           src_id, src_data);
4828                                 break;
4829                         }
4830                         afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4831                         queue_hdmi = true;
4832                         DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4833                         break;
4834                 case 96:
4835                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4836                         WREG32(SRBM_INT_ACK, 0x1);
4837                         break;
4838                 case 124: /* UVD */
4839                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4840                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4841                         break;
4842                 case 146:
4843                 case 147:
4844                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4845                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4846                         /* reset addr and status */
4847                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4848                         if (addr == 0x0 && status == 0x0)
4849                                 break;
4850                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4851                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4852                                 addr);
4853                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4854                                 status);
4855                         cayman_vm_decode_fault(rdev, status, addr);
4856                         break;
4857                 case 176: /* CP_INT in ring buffer */
4858                 case 177: /* CP_INT in IB1 */
4859                 case 178: /* CP_INT in IB2 */
4860                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4862                         break;
4863                 case 181: /* CP EOP event */
4864                         DRM_DEBUG("IH: CP EOP\n");
4865                         if (rdev->family >= CHIP_CAYMAN) {
4866                                 switch (src_data) {
4867                                 case 0:
4868                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869                                         break;
4870                                 case 1:
4871                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872                                         break;
4873                                 case 2:
4874                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875                                         break;
4876                                 }
4877                         } else
4878                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4879                         break;
4880                 case 224: /* DMA trap event */
4881                         DRM_DEBUG("IH: DMA trap\n");
4882                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883                         break;
4884                 case 230: /* thermal low to high */
4885                         DRM_DEBUG("IH: thermal low to high\n");
4886                         rdev->pm.dpm.thermal.high_to_low = false;
4887                         queue_thermal = true;
4888                         break;
4889                 case 231: /* thermal high to low */
4890                         DRM_DEBUG("IH: thermal high to low\n");
4891                         rdev->pm.dpm.thermal.high_to_low = true;
4892                         queue_thermal = true;
4893                         break;
4894                 case 233: /* GUI IDLE */
4895                         DRM_DEBUG("IH: GUI idle\n");
4896                         break;
4897                 case 244: /* DMA trap event */
4898                         if (rdev->family >= CHIP_CAYMAN) {
4899                                 DRM_DEBUG("IH: DMA1 trap\n");
4900                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901                         }
4902                         break;
4903                 default:
4904                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905                         break;
4906                 }
4907
4908                 /* wptr/rptr are in bytes! */
4909                 rptr += 16;
4910                 rptr &= rdev->ih.ptr_mask;
4911                 WREG32(IH_RB_RPTR, rptr);
4912         }
4913         if (queue_dp)
4914                 schedule_work(&rdev->dp_work);
4915         if (queue_hotplug)
4916                 schedule_delayed_work(&rdev->hotplug_work, 0);
4917         if (queue_hdmi)
4918                 schedule_work(&rdev->audio_work);
4919         if (queue_thermal && rdev->pm.dpm_enabled)
4920                 schedule_work(&rdev->pm.dpm.thermal.work);
4921         rdev->ih.rptr = rptr;
4922         atomic_set(&rdev->ih.lock, 0);
4923
4924         /* make sure wptr hasn't changed while processing */
4925         wptr = evergreen_get_ih_wptr(rdev);
4926         if (wptr != rptr)
4927                 goto restart_ih;
4928
4929         return IRQ_HANDLED;
4930 }
4931
4932 static void evergreen_uvd_init(struct radeon_device *rdev)
4933 {
4934         int r;
4935
4936         if (!rdev->has_uvd)
4937                 return;
4938
4939         r = radeon_uvd_init(rdev);
4940         if (r) {
4941                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4942                 /*
4943                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4944                  * to early fails uvd_v2_2_resume() and thus nothing happens
4945                  * there. So it is pointless to try to go through that code
4946                  * hence why we disable uvd here.
4947                  */
4948                 rdev->has_uvd = false;
4949                 return;
4950         }
4951         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4952         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4953 }
4954
4955 static void evergreen_uvd_start(struct radeon_device *rdev)
4956 {
4957         int r;
4958
4959         if (!rdev->has_uvd)
4960                 return;
4961
4962         r = uvd_v2_2_resume(rdev);
4963         if (r) {
4964                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4965                 goto error;
4966         }
4967         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4968         if (r) {
4969                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4970                 goto error;
4971         }
4972         return;
4973
4974 error:
4975         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4976 }
4977
4978 static void evergreen_uvd_resume(struct radeon_device *rdev)
4979 {
4980         struct radeon_ring *ring;
4981         int r;
4982
4983         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4984                 return;
4985
4986         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4987         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4988         if (r) {
4989                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4990                 return;
4991         }
4992         r = uvd_v1_0_init(rdev);
4993         if (r) {
4994                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4995                 return;
4996         }
4997 }
4998
4999 static int evergreen_startup(struct radeon_device *rdev)
5000 {
5001         struct radeon_ring *ring;
5002         int r;
5003
5004         /* enable pcie gen2 link */
5005         evergreen_pcie_gen2_enable(rdev);
5006         /* enable aspm */
5007         evergreen_program_aspm(rdev);
5008
5009         /* scratch needs to be initialized before MC */
5010         r = r600_vram_scratch_init(rdev);
5011         if (r)
5012                 return r;
5013
5014         evergreen_mc_program(rdev);
5015
5016         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5017                 r = ni_mc_load_microcode(rdev);
5018                 if (r) {
5019                         DRM_ERROR("Failed to load MC firmware!\n");
5020                         return r;
5021                 }
5022         }
5023
5024         if (rdev->flags & RADEON_IS_AGP) {
5025                 evergreen_agp_enable(rdev);
5026         } else {
5027                 r = evergreen_pcie_gart_enable(rdev);
5028                 if (r)
5029                         return r;
5030         }
5031         evergreen_gpu_init(rdev);
5032
5033         /* allocate rlc buffers */
5034         if (rdev->flags & RADEON_IS_IGP) {
5035                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5036                 rdev->rlc.reg_list_size =
5037                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5038                 rdev->rlc.cs_data = evergreen_cs_data;
5039                 r = sumo_rlc_init(rdev);
5040                 if (r) {
5041                         DRM_ERROR("Failed to init rlc BOs!\n");
5042                         return r;
5043                 }
5044         }
5045
5046         /* allocate wb buffer */
5047         r = radeon_wb_init(rdev);
5048         if (r)
5049                 return r;
5050
5051         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5052         if (r) {
5053                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5054                 return r;
5055         }
5056
5057         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5058         if (r) {
5059                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5060                 return r;
5061         }
5062
5063         evergreen_uvd_start(rdev);
5064
5065         /* Enable IRQ */
5066         if (!rdev->irq.installed) {
5067                 r = radeon_irq_kms_init(rdev);
5068                 if (r)
5069                         return r;
5070         }
5071
5072         r = r600_irq_init(rdev);
5073         if (r) {
5074                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5075                 radeon_irq_kms_fini(rdev);
5076                 return r;
5077         }
5078         evergreen_irq_set(rdev);
5079
5080         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5081         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5082                              RADEON_CP_PACKET2);
5083         if (r)
5084                 return r;
5085
5086         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5087         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5088                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5089         if (r)
5090                 return r;
5091
5092         r = evergreen_cp_load_microcode(rdev);
5093         if (r)
5094                 return r;
5095         r = evergreen_cp_resume(rdev);
5096         if (r)
5097                 return r;
5098         r = r600_dma_resume(rdev);
5099         if (r)
5100                 return r;
5101
5102         evergreen_uvd_resume(rdev);
5103
5104         r = radeon_ib_pool_init(rdev);
5105         if (r) {
5106                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5107                 return r;
5108         }
5109
5110         r = radeon_audio_init(rdev);
5111         if (r) {
5112                 DRM_ERROR("radeon: audio init failed\n");
5113                 return r;
5114         }
5115
5116         return 0;
5117 }
5118
5119 int evergreen_resume(struct radeon_device *rdev)
5120 {
5121         int r;
5122
5123         /* reset the asic, the gfx blocks are often in a bad state
5124          * after the driver is unloaded or after a resume
5125          */
5126         if (radeon_asic_reset(rdev))
5127                 dev_warn(rdev->dev, "GPU reset failed !\n");
5128         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5129          * posting will perform necessary task to bring back GPU into good
5130          * shape.
5131          */
5132         /* post card */
5133         atom_asic_init(rdev->mode_info.atom_context);
5134
5135         /* init golden registers */
5136         evergreen_init_golden_registers(rdev);
5137
5138         if (rdev->pm.pm_method == PM_METHOD_DPM)
5139                 radeon_pm_resume(rdev);
5140
5141         rdev->accel_working = true;
5142         r = evergreen_startup(rdev);
5143         if (r) {
5144                 DRM_ERROR("evergreen startup failed on resume\n");
5145                 rdev->accel_working = false;
5146                 return r;
5147         }
5148
5149         return r;
5150
5151 }
5152
5153 int evergreen_suspend(struct radeon_device *rdev)
5154 {
5155         radeon_pm_suspend(rdev);
5156         radeon_audio_fini(rdev);
5157         if (rdev->has_uvd) {
5158                 uvd_v1_0_fini(rdev);
5159                 radeon_uvd_suspend(rdev);
5160         }
5161         r700_cp_stop(rdev);
5162         r600_dma_stop(rdev);
5163         evergreen_irq_suspend(rdev);
5164         radeon_wb_disable(rdev);
5165         evergreen_pcie_gart_disable(rdev);
5166
5167         return 0;
5168 }
5169
5170 /* Plan is to move initialization in that function and use
5171  * helper function so that radeon_device_init pretty much
5172  * do nothing more than calling asic specific function. This
5173  * should also allow to remove a bunch of callback function
5174  * like vram_info.
5175  */
5176 int evergreen_init(struct radeon_device *rdev)
5177 {
5178         int r;
5179
5180         /* Read BIOS */
5181         if (!radeon_get_bios(rdev)) {
5182                 if (ASIC_IS_AVIVO(rdev))
5183                         return -EINVAL;
5184         }
5185         /* Must be an ATOMBIOS */
5186         if (!rdev->is_atom_bios) {
5187                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5188                 return -EINVAL;
5189         }
5190         r = radeon_atombios_init(rdev);
5191         if (r)
5192                 return r;
5193         /* reset the asic, the gfx blocks are often in a bad state
5194          * after the driver is unloaded or after a resume
5195          */
5196         if (radeon_asic_reset(rdev))
5197                 dev_warn(rdev->dev, "GPU reset failed !\n");
5198         /* Post card if necessary */
5199         if (!radeon_card_posted(rdev)) {
5200                 if (!rdev->bios) {
5201                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5202                         return -EINVAL;
5203                 }
5204                 DRM_INFO("GPU not posted. posting now...\n");
5205                 atom_asic_init(rdev->mode_info.atom_context);
5206         }
5207         /* init golden registers */
5208         evergreen_init_golden_registers(rdev);
5209         /* Initialize scratch registers */
5210         r600_scratch_init(rdev);
5211         /* Initialize surface registers */
5212         radeon_surface_init(rdev);
5213         /* Initialize clocks */
5214         radeon_get_clock_info(rdev->ddev);
5215         /* Fence driver */
5216         r = radeon_fence_driver_init(rdev);
5217         if (r)
5218                 return r;
5219         /* initialize AGP */
5220         if (rdev->flags & RADEON_IS_AGP) {
5221                 r = radeon_agp_init(rdev);
5222                 if (r)
5223                         radeon_agp_disable(rdev);
5224         }
5225         /* initialize memory controller */
5226         r = evergreen_mc_init(rdev);
5227         if (r)
5228                 return r;
5229         /* Memory manager */
5230         r = radeon_bo_init(rdev);
5231         if (r)
5232                 return r;
5233
5234         if (ASIC_IS_DCE5(rdev)) {
5235                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5236                         r = ni_init_microcode(rdev);
5237                         if (r) {
5238                                 DRM_ERROR("Failed to load firmware!\n");
5239                                 return r;
5240                         }
5241                 }
5242         } else {
5243                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5244                         r = r600_init_microcode(rdev);
5245                         if (r) {
5246                                 DRM_ERROR("Failed to load firmware!\n");
5247                                 /*(DEBLOBBED)*/
5248                         }
5249                 }
5250         }
5251
5252         /* Initialize power management */
5253         radeon_pm_init(rdev);
5254
5255         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5256         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5257
5258         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5259         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5260
5261         evergreen_uvd_init(rdev);
5262
5263         rdev->ih.ring_obj = NULL;
5264         r600_ih_ring_init(rdev, 64 * 1024);
5265
5266         r = r600_pcie_gart_init(rdev);
5267         if (r)
5268                 return r;
5269
5270         rdev->accel_working = true;
5271         r = evergreen_startup(rdev);
5272         if (r) {
5273                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5274                 r700_cp_fini(rdev);
5275                 r600_dma_fini(rdev);
5276                 r600_irq_fini(rdev);
5277                 if (rdev->flags & RADEON_IS_IGP)
5278                         sumo_rlc_fini(rdev);
5279                 radeon_wb_fini(rdev);
5280                 radeon_ib_pool_fini(rdev);
5281                 radeon_irq_kms_fini(rdev);
5282                 evergreen_pcie_gart_fini(rdev);
5283                 rdev->accel_working = false;
5284         }
5285
5286         /* Don't start up if the MC ucode is missing on BTC parts.
5287          * The default clocks and voltages before the MC ucode
5288          * is loaded are not suffient for advanced operations.
5289          */
5290         if (ASIC_IS_DCE5(rdev)) {
5291                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5292                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5293                         return -EINVAL;
5294                 }
5295         }
5296
5297         return 0;
5298 }
5299
5300 void evergreen_fini(struct radeon_device *rdev)
5301 {
5302         radeon_pm_fini(rdev);
5303         radeon_audio_fini(rdev);
5304         r700_cp_fini(rdev);
5305         r600_dma_fini(rdev);
5306         r600_irq_fini(rdev);
5307         if (rdev->flags & RADEON_IS_IGP)
5308                 sumo_rlc_fini(rdev);
5309         radeon_wb_fini(rdev);
5310         radeon_ib_pool_fini(rdev);
5311         radeon_irq_kms_fini(rdev);
5312         uvd_v1_0_fini(rdev);
5313         radeon_uvd_fini(rdev);
5314         evergreen_pcie_gart_fini(rdev);
5315         r600_vram_scratch_fini(rdev);
5316         radeon_gem_fini(rdev);
5317         radeon_fence_driver_fini(rdev);
5318         radeon_agp_fini(rdev);
5319         radeon_bo_fini(rdev);
5320         radeon_atombios_fini(rdev);
5321         kfree(rdev->bios);
5322         rdev->bios = NULL;
5323 }
5324
5325 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5326 {
5327         u32 link_width_cntl, speed_cntl;
5328
5329         if (radeon_pcie_gen2 == 0)
5330                 return;
5331
5332         if (rdev->flags & RADEON_IS_IGP)
5333                 return;
5334
5335         if (!(rdev->flags & RADEON_IS_PCIE))
5336                 return;
5337
5338         /* x2 cards have a special sequence */
5339         if (ASIC_IS_X2(rdev))
5340                 return;
5341
5342         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5343                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5344                 return;
5345
5346         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5347         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5348                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5349                 return;
5350         }
5351
5352         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5353
5354         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5355             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5356
5357                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5358                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5359                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5360
5361                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5362                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5363                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5364
5365                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5366                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5367                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5368
5369                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5370                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5371                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5372
5373                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5374                 speed_cntl |= LC_GEN2_EN_STRAP;
5375                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5376
5377         } else {
5378                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5379                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5380                 if (1)
5381                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5382                 else
5383                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5384                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5385         }
5386 }
5387
5388 void evergreen_program_aspm(struct radeon_device *rdev)
5389 {
5390         u32 data, orig;
5391         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5392         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5393         /* fusion_platform = true
5394          * if the system is a fusion system
5395          * (APU or DGPU in a fusion system).
5396          * todo: check if the system is a fusion platform.
5397          */
5398         bool fusion_platform = false;
5399
5400         if (radeon_aspm == 0)
5401                 return;
5402
5403         if (!(rdev->flags & RADEON_IS_PCIE))
5404                 return;
5405
5406         switch (rdev->family) {
5407         case CHIP_CYPRESS:
5408         case CHIP_HEMLOCK:
5409         case CHIP_JUNIPER:
5410         case CHIP_REDWOOD:
5411         case CHIP_CEDAR:
5412         case CHIP_SUMO:
5413         case CHIP_SUMO2:
5414         case CHIP_PALM:
5415         case CHIP_ARUBA:
5416                 disable_l0s = true;
5417                 break;
5418         default:
5419                 disable_l0s = false;
5420                 break;
5421         }
5422
5423         if (rdev->flags & RADEON_IS_IGP)
5424                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5425
5426         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5427         if (fusion_platform)
5428                 data &= ~MULTI_PIF;
5429         else
5430                 data |= MULTI_PIF;
5431         if (data != orig)
5432                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5433
5434         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5435         if (fusion_platform)
5436                 data &= ~MULTI_PIF;
5437         else
5438                 data |= MULTI_PIF;
5439         if (data != orig)
5440                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5441
5442         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5443         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5444         if (!disable_l0s) {
5445                 if (rdev->family >= CHIP_BARTS)
5446                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5447                 else
5448                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5449         }
5450
5451         if (!disable_l1) {
5452                 if (rdev->family >= CHIP_BARTS)
5453                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5454                 else
5455                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5456
5457                 if (!disable_plloff_in_l1) {
5458                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5459                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5460                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5461                         if (data != orig)
5462                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5463
5464                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5465                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5466                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5467                         if (data != orig)
5468                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5469
5470                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5471                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5472                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5473                         if (data != orig)
5474                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5475
5476                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5477                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5478                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5479                         if (data != orig)
5480                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5481
5482                         if (rdev->family >= CHIP_BARTS) {
5483                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5484                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5485                                 data |= PLL_RAMP_UP_TIME_0(4);
5486                                 if (data != orig)
5487                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5488
5489                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5490                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5491                                 data |= PLL_RAMP_UP_TIME_1(4);
5492                                 if (data != orig)
5493                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5494
5495                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5496                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5497                                 data |= PLL_RAMP_UP_TIME_0(4);
5498                                 if (data != orig)
5499                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5500
5501                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5502                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5503                                 data |= PLL_RAMP_UP_TIME_1(4);
5504                                 if (data != orig)
5505                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5506                         }
5507
5508                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5509                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5510                         data |= LC_DYN_LANES_PWR_STATE(3);
5511                         if (data != orig)
5512                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5513
5514                         if (rdev->family >= CHIP_BARTS) {
5515                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5516                                 data &= ~LS2_EXIT_TIME_MASK;
5517                                 data |= LS2_EXIT_TIME(1);
5518                                 if (data != orig)
5519                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5520
5521                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5522                                 data &= ~LS2_EXIT_TIME_MASK;
5523                                 data |= LS2_EXIT_TIME(1);
5524                                 if (data != orig)
5525                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5526                         }
5527                 }
5528         }
5529
5530         /* evergreen parts only */
5531         if (rdev->family < CHIP_BARTS)
5532                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5533
5534         if (pcie_lc_cntl != pcie_lc_cntl_old)
5535                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5536 }