drivers/gpu/drm/radeon/rv770.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28
  29 #include <linux/firmware.h>
  30 #include <linux/pci.h>
  31 #include <linux/slab.h>
  32
  33 #include <drm/drm_device.h>
  34 #include <drm/radeon_drm.h>
  35 #include <drm/drm_fourcc.h>
  36
  37 #include "atom.h"
  38 #include "avivod.h"
  39 #include "radeon.h"
  40 #include "radeon_asic.h"
  41 #include "radeon_audio.h"
  42 #include "rv770d.h"
  43 #include "rv770.h"
  44
  45 #define R700_PFP_UCODE_SIZE 848
  46 #define R700_PM4_UCODE_SIZE 1360
  47
  48 static void rv770_gpu_init(struct radeon_device *rdev);
  49 void rv770_fini(struct radeon_device *rdev);
  50 static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  51 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
  52
  53 int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  54 {
  55         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
  56         int r;
  57
  58         /* RV740 uses evergreen uvd clk programming */
  59         if (rdev->family == CHIP_RV740)
  60                 return evergreen_set_uvd_clocks(rdev, vclk, dclk);
  61
  62         /* bypass vclk and dclk with bclk */
  63         WREG32_P(CG_UPLL_FUNC_CNTL_2,
  64                  VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
  65                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
  66
  67         if (!vclk || !dclk) {
  68                 /* keep the Bypass mode, put PLL to sleep */
  69                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
  70                 return 0;
  71         }
  72
  73         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
  74                                           43663, 0x03FFFFFE, 1, 30, ~0,
  75                                           &fb_div, &vclk_div, &dclk_div);
  76         if (r)
  77                 return r;
  78
  79         fb_div |= 1;
  80         vclk_div -= 1;
  81         dclk_div -= 1;
  82
  83         /* set UPLL_FB_DIV to 0x50000 */
  84         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
  85
  86         /* deassert UPLL_RESET and UPLL_SLEEP */
  87         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
  88
  89         /* assert BYPASS EN and FB_DIV[0] <- ??? why? */
  90         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
  91         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
  92
  93         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
  94         if (r)
  95                 return r;
  96
  97         /* assert PLL_RESET */
  98         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
  99
 100         /* set the required FB_DIV, REF_DIV, Post divder values */
 101         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
 102         WREG32_P(CG_UPLL_FUNC_CNTL_2,
 103                  UPLL_SW_HILEN(vclk_div >> 1) |
 104                  UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
 105                  UPLL_SW_HILEN2(dclk_div >> 1) |
 106                  UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
 107                  ~UPLL_SW_MASK);
 108
 109         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
 110                  ~UPLL_FB_DIV_MASK);
 111
 112         /* give the PLL some time to settle */
 113         mdelay(15);
 114
 115         /* deassert PLL_RESET */
 116         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
 117
 118         mdelay(15);
 119
 120         /* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
 121         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
 122         WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
 123
 124         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
 125         if (r)
 126                 return r;
 127
 128         /* switch VCLK and DCLK selection */
 129         WREG32_P(CG_UPLL_FUNC_CNTL_2,
 130                  VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
 131                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
 132
 133         mdelay(100);
 134
 135         return 0;
 136 }
 137
 138 static const u32 r7xx_golden_registers[] =
 139 {
 140         0x8d00, 0xffffffff, 0x0e0e0074,
 141         0x8d04, 0xffffffff, 0x013a2b34,
 142         0x9508, 0xffffffff, 0x00000002,
 143         0x8b20, 0xffffffff, 0,
 144         0x88c4, 0xffffffff, 0x000000c2,
 145         0x28350, 0xffffffff, 0,
 146         0x9058, 0xffffffff, 0x0fffc40f,
 147         0x240c, 0xffffffff, 0x00000380,
 148         0x733c, 0xffffffff, 0x00000002,
 149         0x2650, 0x00040000, 0,
 150         0x20bc, 0x00040000, 0,
 151         0x7300, 0xffffffff, 0x001000f0
 152 };
 153
 154 static const u32 r7xx_golden_dyn_gpr_registers[] =
 155 {
 156         0x8db0, 0xffffffff, 0x98989898,
 157         0x8db4, 0xffffffff, 0x98989898,
 158         0x8db8, 0xffffffff, 0x98989898,
 159         0x8dbc, 0xffffffff, 0x98989898,
 160         0x8dc0, 0xffffffff, 0x98989898,
 161         0x8dc4, 0xffffffff, 0x98989898,
 162         0x8dc8, 0xffffffff, 0x98989898,
 163         0x8dcc, 0xffffffff, 0x98989898,
 164         0x88c4, 0xffffffff, 0x00000082
 165 };
 166
 167 static const u32 rv770_golden_registers[] =
 168 {
 169         0x562c, 0xffffffff, 0,
 170         0x3f90, 0xffffffff, 0,
 171         0x9148, 0xffffffff, 0,
 172         0x3f94, 0xffffffff, 0,
 173         0x914c, 0xffffffff, 0,
 174         0x9698, 0x18000000, 0x18000000
 175 };
 176
 177 static const u32 rv770ce_golden_registers[] =
 178 {
 179         0x562c, 0xffffffff, 0,
 180         0x3f90, 0xffffffff, 0x00cc0000,
 181         0x9148, 0xffffffff, 0x00cc0000,
 182         0x3f94, 0xffffffff, 0x00cc0000,
 183         0x914c, 0xffffffff, 0x00cc0000,
 184         0x9b7c, 0xffffffff, 0x00fa0000,
 185         0x3f8c, 0xffffffff, 0x00fa0000,
 186         0x9698, 0x18000000, 0x18000000
 187 };
 188
 189 static const u32 rv770_mgcg_init[] =
 190 {
 191         0x8bcc, 0xffffffff, 0x130300f9,
 192         0x5448, 0xffffffff, 0x100,
 193         0x55e4, 0xffffffff, 0x100,
 194         0x160c, 0xffffffff, 0x100,
 195         0x5644, 0xffffffff, 0x100,
 196         0xc164, 0xffffffff, 0x100,
 197         0x8a18, 0xffffffff, 0x100,
 198         0x897c, 0xffffffff, 0x8000100,
 199         0x8b28, 0xffffffff, 0x3c000100,
 200         0x9144, 0xffffffff, 0x100,
 201         0x9a1c, 0xffffffff, 0x10000,
 202         0x9a50, 0xffffffff, 0x100,
 203         0x9a1c, 0xffffffff, 0x10001,
 204         0x9a50, 0xffffffff, 0x100,
 205         0x9a1c, 0xffffffff, 0x10002,
 206         0x9a50, 0xffffffff, 0x100,
 207         0x9a1c, 0xffffffff, 0x10003,
 208         0x9a50, 0xffffffff, 0x100,
 209         0x9a1c, 0xffffffff, 0x0,
 210         0x9870, 0xffffffff, 0x100,
 211         0x8d58, 0xffffffff, 0x100,
 212         0x9500, 0xffffffff, 0x0,
 213         0x9510, 0xffffffff, 0x100,
 214         0x9500, 0xffffffff, 0x1,
 215         0x9510, 0xffffffff, 0x100,
 216         0x9500, 0xffffffff, 0x2,
 217         0x9510, 0xffffffff, 0x100,
 218         0x9500, 0xffffffff, 0x3,
 219         0x9510, 0xffffffff, 0x100,
 220         0x9500, 0xffffffff, 0x4,
 221         0x9510, 0xffffffff, 0x100,
 222         0x9500, 0xffffffff, 0x5,
 223         0x9510, 0xffffffff, 0x100,
 224         0x9500, 0xffffffff, 0x6,
 225         0x9510, 0xffffffff, 0x100,
 226         0x9500, 0xffffffff, 0x7,
 227         0x9510, 0xffffffff, 0x100,
 228         0x9500, 0xffffffff, 0x8,
 229         0x9510, 0xffffffff, 0x100,
 230         0x9500, 0xffffffff, 0x9,
 231         0x9510, 0xffffffff, 0x100,
 232         0x9500, 0xffffffff, 0x8000,
 233         0x9490, 0xffffffff, 0x0,
 234         0x949c, 0xffffffff, 0x100,
 235         0x9490, 0xffffffff, 0x1,
 236         0x949c, 0xffffffff, 0x100,
 237         0x9490, 0xffffffff, 0x2,
 238         0x949c, 0xffffffff, 0x100,
 239         0x9490, 0xffffffff, 0x3,
 240         0x949c, 0xffffffff, 0x100,
 241         0x9490, 0xffffffff, 0x4,
 242         0x949c, 0xffffffff, 0x100,
 243         0x9490, 0xffffffff, 0x5,
 244         0x949c, 0xffffffff, 0x100,
 245         0x9490, 0xffffffff, 0x6,
 246         0x949c, 0xffffffff, 0x100,
 247         0x9490, 0xffffffff, 0x7,
 248         0x949c, 0xffffffff, 0x100,
 249         0x9490, 0xffffffff, 0x8,
 250         0x949c, 0xffffffff, 0x100,
 251         0x9490, 0xffffffff, 0x9,
 252         0x949c, 0xffffffff, 0x100,
 253         0x9490, 0xffffffff, 0x8000,
 254         0x9604, 0xffffffff, 0x0,
 255         0x9654, 0xffffffff, 0x100,
 256         0x9604, 0xffffffff, 0x1,
 257         0x9654, 0xffffffff, 0x100,
 258         0x9604, 0xffffffff, 0x2,
 259         0x9654, 0xffffffff, 0x100,
 260         0x9604, 0xffffffff, 0x3,
 261         0x9654, 0xffffffff, 0x100,
 262         0x9604, 0xffffffff, 0x4,
 263         0x9654, 0xffffffff, 0x100,
 264         0x9604, 0xffffffff, 0x5,
 265         0x9654, 0xffffffff, 0x100,
 266         0x9604, 0xffffffff, 0x6,
 267         0x9654, 0xffffffff, 0x100,
 268         0x9604, 0xffffffff, 0x7,
 269         0x9654, 0xffffffff, 0x100,
 270         0x9604, 0xffffffff, 0x8,
 271         0x9654, 0xffffffff, 0x100,
 272         0x9604, 0xffffffff, 0x9,
 273         0x9654, 0xffffffff, 0x100,
 274         0x9604, 0xffffffff, 0x80000000,
 275         0x9030, 0xffffffff, 0x100,
 276         0x9034, 0xffffffff, 0x100,
 277         0x9038, 0xffffffff, 0x100,
 278         0x903c, 0xffffffff, 0x100,
 279         0x9040, 0xffffffff, 0x100,
 280         0xa200, 0xffffffff, 0x100,
 281         0xa204, 0xffffffff, 0x100,
 282         0xa208, 0xffffffff, 0x100,
 283         0xa20c, 0xffffffff, 0x100,
 284         0x971c, 0xffffffff, 0x100,
 285         0x915c, 0xffffffff, 0x00020001,
 286         0x9160, 0xffffffff, 0x00040003,
 287         0x916c, 0xffffffff, 0x00060005,
 288         0x9170, 0xffffffff, 0x00080007,
 289         0x9174, 0xffffffff, 0x000a0009,
 290         0x9178, 0xffffffff, 0x000c000b,
 291         0x917c, 0xffffffff, 0x000e000d,
 292         0x9180, 0xffffffff, 0x0010000f,
 293         0x918c, 0xffffffff, 0x00120011,
 294         0x9190, 0xffffffff, 0x00140013,
 295         0x9194, 0xffffffff, 0x00020001,
 296         0x9198, 0xffffffff, 0x00040003,
 297         0x919c, 0xffffffff, 0x00060005,
 298         0x91a8, 0xffffffff, 0x00080007,
 299         0x91ac, 0xffffffff, 0x000a0009,
 300         0x91b0, 0xffffffff, 0x000c000b,
 301         0x91b4, 0xffffffff, 0x000e000d,
 302         0x91b8, 0xffffffff, 0x0010000f,
 303         0x91c4, 0xffffffff, 0x00120011,
 304         0x91c8, 0xffffffff, 0x00140013,
 305         0x91cc, 0xffffffff, 0x00020001,
 306         0x91d0, 0xffffffff, 0x00040003,
 307         0x91d4, 0xffffffff, 0x00060005,
 308         0x91e0, 0xffffffff, 0x00080007,
 309         0x91e4, 0xffffffff, 0x000a0009,
 310         0x91e8, 0xffffffff, 0x000c000b,
 311         0x91ec, 0xffffffff, 0x00020001,
 312         0x91f0, 0xffffffff, 0x00040003,
 313         0x91f4, 0xffffffff, 0x00060005,
 314         0x9200, 0xffffffff, 0x00080007,
 315         0x9204, 0xffffffff, 0x000a0009,
 316         0x9208, 0xffffffff, 0x000c000b,
 317         0x920c, 0xffffffff, 0x000e000d,
 318         0x9210, 0xffffffff, 0x0010000f,
 319         0x921c, 0xffffffff, 0x00120011,
 320         0x9220, 0xffffffff, 0x00140013,
 321         0x9224, 0xffffffff, 0x00020001,
 322         0x9228, 0xffffffff, 0x00040003,
 323         0x922c, 0xffffffff, 0x00060005,
 324         0x9238, 0xffffffff, 0x00080007,
 325         0x923c, 0xffffffff, 0x000a0009,
 326         0x9240, 0xffffffff, 0x000c000b,
 327         0x9244, 0xffffffff, 0x000e000d,
 328         0x9248, 0xffffffff, 0x0010000f,
 329         0x9254, 0xffffffff, 0x00120011,
 330         0x9258, 0xffffffff, 0x00140013,
 331         0x925c, 0xffffffff, 0x00020001,
 332         0x9260, 0xffffffff, 0x00040003,
 333         0x9264, 0xffffffff, 0x00060005,
 334         0x9270, 0xffffffff, 0x00080007,
 335         0x9274, 0xffffffff, 0x000a0009,
 336         0x9278, 0xffffffff, 0x000c000b,
 337         0x927c, 0xffffffff, 0x000e000d,
 338         0x9280, 0xffffffff, 0x0010000f,
 339         0x928c, 0xffffffff, 0x00120011,
 340         0x9290, 0xffffffff, 0x00140013,
 341         0x9294, 0xffffffff, 0x00020001,
 342         0x929c, 0xffffffff, 0x00040003,
 343         0x92a0, 0xffffffff, 0x00060005,
 344         0x92a4, 0xffffffff, 0x00080007
 345 };
 346
 347 static const u32 rv710_golden_registers[] =
 348 {
 349         0x3f90, 0x00ff0000, 0x00fc0000,
 350         0x9148, 0x00ff0000, 0x00fc0000,
 351         0x3f94, 0x00ff0000, 0x00fc0000,
 352         0x914c, 0x00ff0000, 0x00fc0000,
 353         0xb4c, 0x00000020, 0x00000020,
 354         0xa180, 0xffffffff, 0x00003f3f
 355 };
 356
 357 static const u32 rv710_mgcg_init[] =
 358 {
 359         0x8bcc, 0xffffffff, 0x13030040,
 360         0x5448, 0xffffffff, 0x100,
 361         0x55e4, 0xffffffff, 0x100,
 362         0x160c, 0xffffffff, 0x100,
 363         0x5644, 0xffffffff, 0x100,
 364         0xc164, 0xffffffff, 0x100,
 365         0x8a18, 0xffffffff, 0x100,
 366         0x897c, 0xffffffff, 0x8000100,
 367         0x8b28, 0xffffffff, 0x3c000100,
 368         0x9144, 0xffffffff, 0x100,
 369         0x9a1c, 0xffffffff, 0x10000,
 370         0x9a50, 0xffffffff, 0x100,
 371         0x9a1c, 0xffffffff, 0x0,
 372         0x9870, 0xffffffff, 0x100,
 373         0x8d58, 0xffffffff, 0x100,
 374         0x9500, 0xffffffff, 0x0,
 375         0x9510, 0xffffffff, 0x100,
 376         0x9500, 0xffffffff, 0x1,
 377         0x9510, 0xffffffff, 0x100,
 378         0x9500, 0xffffffff, 0x8000,
 379         0x9490, 0xffffffff, 0x0,
 380         0x949c, 0xffffffff, 0x100,
 381         0x9490, 0xffffffff, 0x1,
 382         0x949c, 0xffffffff, 0x100,
 383         0x9490, 0xffffffff, 0x8000,
 384         0x9604, 0xffffffff, 0x0,
 385         0x9654, 0xffffffff, 0x100,
 386         0x9604, 0xffffffff, 0x1,
 387         0x9654, 0xffffffff, 0x100,
 388         0x9604, 0xffffffff, 0x80000000,
 389         0x9030, 0xffffffff, 0x100,
 390         0x9034, 0xffffffff, 0x100,
 391         0x9038, 0xffffffff, 0x100,
 392         0x903c, 0xffffffff, 0x100,
 393         0x9040, 0xffffffff, 0x100,
 394         0xa200, 0xffffffff, 0x100,
 395         0xa204, 0xffffffff, 0x100,
 396         0xa208, 0xffffffff, 0x100,
 397         0xa20c, 0xffffffff, 0x100,
 398         0x971c, 0xffffffff, 0x100,
 399         0x915c, 0xffffffff, 0x00020001,
 400         0x9174, 0xffffffff, 0x00000003,
 401         0x9178, 0xffffffff, 0x00050001,
 402         0x917c, 0xffffffff, 0x00030002,
 403         0x918c, 0xffffffff, 0x00000004,
 404         0x9190, 0xffffffff, 0x00070006,
 405         0x9194, 0xffffffff, 0x00050001,
 406         0x9198, 0xffffffff, 0x00030002,
 407         0x91a8, 0xffffffff, 0x00000004,
 408         0x91ac, 0xffffffff, 0x00070006,
 409         0x91e8, 0xffffffff, 0x00000001,
 410         0x9294, 0xffffffff, 0x00000001,
 411         0x929c, 0xffffffff, 0x00000002,
 412         0x92a0, 0xffffffff, 0x00040003,
 413         0x9150, 0xffffffff, 0x4d940000
 414 };
 415
 416 static const u32 rv730_golden_registers[] =
 417 {
 418         0x3f90, 0x00ff0000, 0x00f00000,
 419         0x9148, 0x00ff0000, 0x00f00000,
 420         0x3f94, 0x00ff0000, 0x00f00000,
 421         0x914c, 0x00ff0000, 0x00f00000,
 422         0x900c, 0xffffffff, 0x003b033f,
 423         0xb4c, 0x00000020, 0x00000020,
 424         0xa180, 0xffffffff, 0x00003f3f
 425 };
 426
 427 static const u32 rv730_mgcg_init[] =
 428 {
 429         0x8bcc, 0xffffffff, 0x130300f9,
 430         0x5448, 0xffffffff, 0x100,
 431         0x55e4, 0xffffffff, 0x100,
 432         0x160c, 0xffffffff, 0x100,
 433         0x5644, 0xffffffff, 0x100,
 434         0xc164, 0xffffffff, 0x100,
 435         0x8a18, 0xffffffff, 0x100,
 436         0x897c, 0xffffffff, 0x8000100,
 437         0x8b28, 0xffffffff, 0x3c000100,
 438         0x9144, 0xffffffff, 0x100,
 439         0x9a1c, 0xffffffff, 0x10000,
 440         0x9a50, 0xffffffff, 0x100,
 441         0x9a1c, 0xffffffff, 0x10001,
 442         0x9a50, 0xffffffff, 0x100,
 443         0x9a1c, 0xffffffff, 0x0,
 444         0x9870, 0xffffffff, 0x100,
 445         0x8d58, 0xffffffff, 0x100,
 446         0x9500, 0xffffffff, 0x0,
 447         0x9510, 0xffffffff, 0x100,
 448         0x9500, 0xffffffff, 0x1,
 449         0x9510, 0xffffffff, 0x100,
 450         0x9500, 0xffffffff, 0x2,
 451         0x9510, 0xffffffff, 0x100,
 452         0x9500, 0xffffffff, 0x3,
 453         0x9510, 0xffffffff, 0x100,
 454         0x9500, 0xffffffff, 0x4,
 455         0x9510, 0xffffffff, 0x100,
 456         0x9500, 0xffffffff, 0x5,
 457         0x9510, 0xffffffff, 0x100,
 458         0x9500, 0xffffffff, 0x6,
 459         0x9510, 0xffffffff, 0x100,
 460         0x9500, 0xffffffff, 0x7,
 461         0x9510, 0xffffffff, 0x100,
 462         0x9500, 0xffffffff, 0x8000,
 463         0x9490, 0xffffffff, 0x0,
 464         0x949c, 0xffffffff, 0x100,
 465         0x9490, 0xffffffff, 0x1,
 466         0x949c, 0xffffffff, 0x100,
 467         0x9490, 0xffffffff, 0x2,
 468         0x949c, 0xffffffff, 0x100,
 469         0x9490, 0xffffffff, 0x3,
 470         0x949c, 0xffffffff, 0x100,
 471         0x9490, 0xffffffff, 0x4,
 472         0x949c, 0xffffffff, 0x100,
 473         0x9490, 0xffffffff, 0x5,
 474         0x949c, 0xffffffff, 0x100,
 475         0x9490, 0xffffffff, 0x6,
 476         0x949c, 0xffffffff, 0x100,
 477         0x9490, 0xffffffff, 0x7,
 478         0x949c, 0xffffffff, 0x100,
 479         0x9490, 0xffffffff, 0x8000,
 480         0x9604, 0xffffffff, 0x0,
 481         0x9654, 0xffffffff, 0x100,
 482         0x9604, 0xffffffff, 0x1,
 483         0x9654, 0xffffffff, 0x100,
 484         0x9604, 0xffffffff, 0x2,
 485         0x9654, 0xffffffff, 0x100,
 486         0x9604, 0xffffffff, 0x3,
 487         0x9654, 0xffffffff, 0x100,
 488         0x9604, 0xffffffff, 0x4,
 489         0x9654, 0xffffffff, 0x100,
 490         0x9604, 0xffffffff, 0x5,
 491         0x9654, 0xffffffff, 0x100,
 492         0x9604, 0xffffffff, 0x6,
 493         0x9654, 0xffffffff, 0x100,
 494         0x9604, 0xffffffff, 0x7,
 495         0x9654, 0xffffffff, 0x100,
 496         0x9604, 0xffffffff, 0x80000000,
 497         0x9030, 0xffffffff, 0x100,
 498         0x9034, 0xffffffff, 0x100,
 499         0x9038, 0xffffffff, 0x100,
 500         0x903c, 0xffffffff, 0x100,
 501         0x9040, 0xffffffff, 0x100,
 502         0xa200, 0xffffffff, 0x100,
 503         0xa204, 0xffffffff, 0x100,
 504         0xa208, 0xffffffff, 0x100,
 505         0xa20c, 0xffffffff, 0x100,
 506         0x971c, 0xffffffff, 0x100,
 507         0x915c, 0xffffffff, 0x00020001,
 508         0x916c, 0xffffffff, 0x00040003,
 509         0x9170, 0xffffffff, 0x00000005,
 510         0x9178, 0xffffffff, 0x00050001,
 511         0x917c, 0xffffffff, 0x00030002,
 512         0x918c, 0xffffffff, 0x00000004,
 513         0x9190, 0xffffffff, 0x00070006,
 514         0x9194, 0xffffffff, 0x00050001,
 515         0x9198, 0xffffffff, 0x00030002,
 516         0x91a8, 0xffffffff, 0x00000004,
 517         0x91ac, 0xffffffff, 0x00070006,
 518         0x91b0, 0xffffffff, 0x00050001,
 519         0x91b4, 0xffffffff, 0x00030002,
 520         0x91c4, 0xffffffff, 0x00000004,
 521         0x91c8, 0xffffffff, 0x00070006,
 522         0x91cc, 0xffffffff, 0x00050001,
 523         0x91d0, 0xffffffff, 0x00030002,
 524         0x91e0, 0xffffffff, 0x00000004,
 525         0x91e4, 0xffffffff, 0x00070006,
 526         0x91e8, 0xffffffff, 0x00000001,
 527         0x91ec, 0xffffffff, 0x00050001,
 528         0x91f0, 0xffffffff, 0x00030002,
 529         0x9200, 0xffffffff, 0x00000004,
 530         0x9204, 0xffffffff, 0x00070006,
 531         0x9208, 0xffffffff, 0x00050001,
 532         0x920c, 0xffffffff, 0x00030002,
 533         0x921c, 0xffffffff, 0x00000004,
 534         0x9220, 0xffffffff, 0x00070006,
 535         0x9224, 0xffffffff, 0x00050001,
 536         0x9228, 0xffffffff, 0x00030002,
 537         0x9238, 0xffffffff, 0x00000004,
 538         0x923c, 0xffffffff, 0x00070006,
 539         0x9240, 0xffffffff, 0x00050001,
 540         0x9244, 0xffffffff, 0x00030002,
 541         0x9254, 0xffffffff, 0x00000004,
 542         0x9258, 0xffffffff, 0x00070006,
 543         0x9294, 0xffffffff, 0x00000001,
 544         0x929c, 0xffffffff, 0x00000002,
 545         0x92a0, 0xffffffff, 0x00040003,
 546         0x92a4, 0xffffffff, 0x00000005
 547 };
 548
 549 static const u32 rv740_golden_registers[] =
 550 {
 551         0x88c4, 0xffffffff, 0x00000082,
 552         0x28a50, 0xfffffffc, 0x00000004,
 553         0x2650, 0x00040000, 0,
 554         0x20bc, 0x00040000, 0,
 555         0x733c, 0xffffffff, 0x00000002,
 556         0x7300, 0xffffffff, 0x001000f0,
 557         0x3f90, 0x00ff0000, 0,
 558         0x9148, 0x00ff0000, 0,
 559         0x3f94, 0x00ff0000, 0,
 560         0x914c, 0x00ff0000, 0,
 561         0x240c, 0xffffffff, 0x00000380,
 562         0x8a14, 0x00000007, 0x00000007,
 563         0x8b24, 0xffffffff, 0x00ff0fff,
 564         0x28a4c, 0xffffffff, 0x00004000,
 565         0xa180, 0xffffffff, 0x00003f3f,
 566         0x8d00, 0xffffffff, 0x0e0e003a,
 567         0x8d04, 0xffffffff, 0x013a0e2a,
 568         0x8c00, 0xffffffff, 0xe400000f,
 569         0x8db0, 0xffffffff, 0x98989898,
 570         0x8db4, 0xffffffff, 0x98989898,
 571         0x8db8, 0xffffffff, 0x98989898,
 572         0x8dbc, 0xffffffff, 0x98989898,
 573         0x8dc0, 0xffffffff, 0x98989898,
 574         0x8dc4, 0xffffffff, 0x98989898,
 575         0x8dc8, 0xffffffff, 0x98989898,
 576         0x8dcc, 0xffffffff, 0x98989898,
 577         0x9058, 0xffffffff, 0x0fffc40f,
 578         0x900c, 0xffffffff, 0x003b033f,
 579         0x28350, 0xffffffff, 0,
 580         0x8cf0, 0x1fffffff, 0x08e00420,
 581         0x9508, 0xffffffff, 0x00000002,
 582         0x88c4, 0xffffffff, 0x000000c2,
 583         0x9698, 0x18000000, 0x18000000
 584 };
 585
 586 static const u32 rv740_mgcg_init[] =
 587 {
 588         0x8bcc, 0xffffffff, 0x13030100,
 589         0x5448, 0xffffffff, 0x100,
 590         0x55e4, 0xffffffff, 0x100,
 591         0x160c, 0xffffffff, 0x100,
 592         0x5644, 0xffffffff, 0x100,
 593         0xc164, 0xffffffff, 0x100,
 594         0x8a18, 0xffffffff, 0x100,
 595         0x897c, 0xffffffff, 0x100,
 596         0x8b28, 0xffffffff, 0x100,
 597         0x9144, 0xffffffff, 0x100,
 598         0x9a1c, 0xffffffff, 0x10000,
 599         0x9a50, 0xffffffff, 0x100,
 600         0x9a1c, 0xffffffff, 0x10001,
 601         0x9a50, 0xffffffff, 0x100,
 602         0x9a1c, 0xffffffff, 0x10002,
 603         0x9a50, 0xffffffff, 0x100,
 604         0x9a1c, 0xffffffff, 0x10003,
 605         0x9a50, 0xffffffff, 0x100,
 606         0x9a1c, 0xffffffff, 0x0,
 607         0x9870, 0xffffffff, 0x100,
 608         0x8d58, 0xffffffff, 0x100,
 609         0x9500, 0xffffffff, 0x0,
 610         0x9510, 0xffffffff, 0x100,
 611         0x9500, 0xffffffff, 0x1,
 612         0x9510, 0xffffffff, 0x100,
 613         0x9500, 0xffffffff, 0x2,
 614         0x9510, 0xffffffff, 0x100,
 615         0x9500, 0xffffffff, 0x3,
 616         0x9510, 0xffffffff, 0x100,
 617         0x9500, 0xffffffff, 0x4,
 618         0x9510, 0xffffffff, 0x100,
 619         0x9500, 0xffffffff, 0x5,
 620         0x9510, 0xffffffff, 0x100,
 621         0x9500, 0xffffffff, 0x6,
 622         0x9510, 0xffffffff, 0x100,
 623         0x9500, 0xffffffff, 0x7,
 624         0x9510, 0xffffffff, 0x100,
 625         0x9500, 0xffffffff, 0x8000,
 626         0x9490, 0xffffffff, 0x0,
 627         0x949c, 0xffffffff, 0x100,
 628         0x9490, 0xffffffff, 0x1,
 629         0x949c, 0xffffffff, 0x100,
 630         0x9490, 0xffffffff, 0x2,
 631         0x949c, 0xffffffff, 0x100,
 632         0x9490, 0xffffffff, 0x3,
 633         0x949c, 0xffffffff, 0x100,
 634         0x9490, 0xffffffff, 0x4,
 635         0x949c, 0xffffffff, 0x100,
 636         0x9490, 0xffffffff, 0x5,
 637         0x949c, 0xffffffff, 0x100,
 638         0x9490, 0xffffffff, 0x6,
 639         0x949c, 0xffffffff, 0x100,
 640         0x9490, 0xffffffff, 0x7,
 641         0x949c, 0xffffffff, 0x100,
 642         0x9490, 0xffffffff, 0x8000,
 643         0x9604, 0xffffffff, 0x0,
 644         0x9654, 0xffffffff, 0x100,
 645         0x9604, 0xffffffff, 0x1,
 646         0x9654, 0xffffffff, 0x100,
 647         0x9604, 0xffffffff, 0x2,
 648         0x9654, 0xffffffff, 0x100,
 649         0x9604, 0xffffffff, 0x3,
 650         0x9654, 0xffffffff, 0x100,
 651         0x9604, 0xffffffff, 0x4,
 652         0x9654, 0xffffffff, 0x100,
 653         0x9604, 0xffffffff, 0x5,
 654         0x9654, 0xffffffff, 0x100,
 655         0x9604, 0xffffffff, 0x6,
 656         0x9654, 0xffffffff, 0x100,
 657         0x9604, 0xffffffff, 0x7,
 658         0x9654, 0xffffffff, 0x100,
 659         0x9604, 0xffffffff, 0x80000000,
 660         0x9030, 0xffffffff, 0x100,
 661         0x9034, 0xffffffff, 0x100,
 662         0x9038, 0xffffffff, 0x100,
 663         0x903c, 0xffffffff, 0x100,
 664         0x9040, 0xffffffff, 0x100,
 665         0xa200, 0xffffffff, 0x100,
 666         0xa204, 0xffffffff, 0x100,
 667         0xa208, 0xffffffff, 0x100,
 668         0xa20c, 0xffffffff, 0x100,
 669         0x971c, 0xffffffff, 0x100,
 670         0x915c, 0xffffffff, 0x00020001,
 671         0x9160, 0xffffffff, 0x00040003,
 672         0x916c, 0xffffffff, 0x00060005,
 673         0x9170, 0xffffffff, 0x00080007,
 674         0x9174, 0xffffffff, 0x000a0009,
 675         0x9178, 0xffffffff, 0x000c000b,
 676         0x917c, 0xffffffff, 0x000e000d,
 677         0x9180, 0xffffffff, 0x0010000f,
 678         0x918c, 0xffffffff, 0x00120011,
 679         0x9190, 0xffffffff, 0x00140013,
 680         0x9194, 0xffffffff, 0x00020001,
 681         0x9198, 0xffffffff, 0x00040003,
 682         0x919c, 0xffffffff, 0x00060005,
 683         0x91a8, 0xffffffff, 0x00080007,
 684         0x91ac, 0xffffffff, 0x000a0009,
 685         0x91b0, 0xffffffff, 0x000c000b,
 686         0x91b4, 0xffffffff, 0x000e000d,
 687         0x91b8, 0xffffffff, 0x0010000f,
 688         0x91c4, 0xffffffff, 0x00120011,
 689         0x91c8, 0xffffffff, 0x00140013,
 690         0x91cc, 0xffffffff, 0x00020001,
 691         0x91d0, 0xffffffff, 0x00040003,
 692         0x91d4, 0xffffffff, 0x00060005,
 693         0x91e0, 0xffffffff, 0x00080007,
 694         0x91e4, 0xffffffff, 0x000a0009,
 695         0x91e8, 0xffffffff, 0x000c000b,
 696         0x91ec, 0xffffffff, 0x00020001,
 697         0x91f0, 0xffffffff, 0x00040003,
 698         0x91f4, 0xffffffff, 0x00060005,
 699         0x9200, 0xffffffff, 0x00080007,
 700         0x9204, 0xffffffff, 0x000a0009,
 701         0x9208, 0xffffffff, 0x000c000b,
 702         0x920c, 0xffffffff, 0x000e000d,
 703         0x9210, 0xffffffff, 0x0010000f,
 704         0x921c, 0xffffffff, 0x00120011,
 705         0x9220, 0xffffffff, 0x00140013,
 706         0x9224, 0xffffffff, 0x00020001,
 707         0x9228, 0xffffffff, 0x00040003,
 708         0x922c, 0xffffffff, 0x00060005,
 709         0x9238, 0xffffffff, 0x00080007,
 710         0x923c, 0xffffffff, 0x000a0009,
 711         0x9240, 0xffffffff, 0x000c000b,
 712         0x9244, 0xffffffff, 0x000e000d,
 713         0x9248, 0xffffffff, 0x0010000f,
 714         0x9254, 0xffffffff, 0x00120011,
 715         0x9258, 0xffffffff, 0x00140013,
 716         0x9294, 0xffffffff, 0x00020001,
 717         0x929c, 0xffffffff, 0x00040003,
 718         0x92a0, 0xffffffff, 0x00060005,
 719         0x92a4, 0xffffffff, 0x00080007
 720 };
 721
 722 static void rv770_init_golden_registers(struct radeon_device *rdev)
 723 {
 724         switch (rdev->family) {
 725         case CHIP_RV770:
 726                 radeon_program_register_sequence(rdev,
 727                                                  r7xx_golden_registers,
 728                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 729                 radeon_program_register_sequence(rdev,
 730                                                  r7xx_golden_dyn_gpr_registers,
 731                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 732                 if (rdev->pdev->device == 0x994e)
 733                         radeon_program_register_sequence(rdev,
 734                                                          rv770ce_golden_registers,
 735                                                          (const u32)ARRAY_SIZE(rv770ce_golden_registers));
 736                 else
 737                         radeon_program_register_sequence(rdev,
 738                                                          rv770_golden_registers,
 739                                                          (const u32)ARRAY_SIZE(rv770_golden_registers));
 740                 radeon_program_register_sequence(rdev,
 741                                                  rv770_mgcg_init,
 742                                                  (const u32)ARRAY_SIZE(rv770_mgcg_init));
 743                 break;
 744         case CHIP_RV730:
 745                 radeon_program_register_sequence(rdev,
 746                                                  r7xx_golden_registers,
 747                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 748                 radeon_program_register_sequence(rdev,
 749                                                  r7xx_golden_dyn_gpr_registers,
 750                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 751                 radeon_program_register_sequence(rdev,
 752                                                  rv730_golden_registers,
 753                                                  (const u32)ARRAY_SIZE(rv730_golden_registers));
 754                 radeon_program_register_sequence(rdev,
 755                                                  rv730_mgcg_init,
 756                                                  (const u32)ARRAY_SIZE(rv730_mgcg_init));
 757                 break;
 758         case CHIP_RV710:
 759                 radeon_program_register_sequence(rdev,
 760                                                  r7xx_golden_registers,
 761                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 762                 radeon_program_register_sequence(rdev,
 763                                                  r7xx_golden_dyn_gpr_registers,
 764                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 765                 radeon_program_register_sequence(rdev,
 766                                                  rv710_golden_registers,
 767                                                  (const u32)ARRAY_SIZE(rv710_golden_registers));
 768                 radeon_program_register_sequence(rdev,
 769                                                  rv710_mgcg_init,
 770                                                  (const u32)ARRAY_SIZE(rv710_mgcg_init));
 771                 break;
 772         case CHIP_RV740:
 773                 radeon_program_register_sequence(rdev,
 774                                                  rv740_golden_registers,
 775                                                  (const u32)ARRAY_SIZE(rv740_golden_registers));
 776                 radeon_program_register_sequence(rdev,
 777                                                  rv740_mgcg_init,
 778                                                  (const u32)ARRAY_SIZE(rv740_mgcg_init));
 779                 break;
 780         default:
 781                 break;
 782         }
 783 }
 784
 785 #define PCIE_BUS_CLK                10000
 786 #define TCLK                        (PCIE_BUS_CLK / 10)
 787
 788 /**
 789  * rv770_get_xclk - get the xclk
 790  *
 791  * @rdev: radeon_device pointer
 792  *
 793  * Returns the reference clock used by the gfx engine
 794  * (r7xx-cayman).
 795  */
 796 u32 rv770_get_xclk(struct radeon_device *rdev)
 797 {
 798         u32 reference_clock = rdev->clock.spll.reference_freq;
 799         u32 tmp = RREG32(CG_CLKPIN_CNTL);
 800
 801         if (tmp & MUX_TCLK_TO_XCLK)
 802                 return TCLK;
 803
 804         if (tmp & XTALIN_DIVIDE)
 805                 return reference_clock / 4;
 806
 807         return reference_clock;
 808 }
 809
 810 void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async)
 811 {
 812         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 813         struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
 814         u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
 815         int i;
 816
 817         /* Lock the graphics update lock */
 818         tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
 819         WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 820
 821         /* flip at hsync for async, default is vsync */
 822         WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
 823                async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
 824         /* update pitch */
 825         WREG32(AVIVO_D1GRPH_PITCH + radeon_crtc->crtc_offset,
 826                fb->pitches[0] / fb->format->cpp[0]);
 827         /* update the scanout addresses */
 828         if (radeon_crtc->crtc_id) {
 829                 WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 830                 WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 831         } else {
 832                 WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 833                 WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 834         }
 835         WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 836                (u32)crtc_base);
 837         WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 838                (u32)crtc_base);
 839
 840         /* Wait for update_pending to go high. */
 841         for (i = 0; i < rdev->usec_timeout; i++) {
 842                 if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
 843                         break;
 844                 udelay(1);
 845         }
 846         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 847
 848         /* Unlock the lock, so double-buffering can take place inside vblank */
 849         tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
 850         WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 851 }
 852
 853 bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
 854 {
 855         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 856
 857         /* Return current update_pending status: */
 858         return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
 859                 AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
 860 }
 861
 862 /* get temperature in millidegrees */
 863 int rv770_get_temp(struct radeon_device *rdev)
 864 {
 865         u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 866                 ASIC_T_SHIFT;
 867         int actual_temp;
 868
 869         if (temp & 0x400)
 870                 actual_temp = -256;
 871         else if (temp & 0x200)
 872                 actual_temp = 255;
 873         else if (temp & 0x100) {
 874                 actual_temp = temp & 0x1ff;
 875                 actual_temp |= ~0x1ff;
 876         } else
 877                 actual_temp = temp & 0xff;
 878
 879         return (actual_temp * 1000) / 2;
 880 }
 881
 882 void rv770_pm_misc(struct radeon_device *rdev)
 883 {
 884         int req_ps_idx = rdev->pm.requested_power_state_index;
 885         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 886         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 887         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 888
 889         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 890                 /* 0xff01 is a flag rather then an actual voltage */
 891                 if (voltage->voltage == 0xff01)
 892                         return;
 893                 if (voltage->voltage != rdev->pm.current_vddc) {
 894                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 895                         rdev->pm.current_vddc = voltage->voltage;
 896                         DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 897                 }
 898         }
 899 }
 900
 901 /*
 902  * GART
 903  */
 904 static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 905 {
 906         u32 tmp;
 907         int r, i;
 908
 909         if (rdev->gart.robj == NULL) {
 910                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 911                 return -EINVAL;
 912         }
 913         r = radeon_gart_table_vram_pin(rdev);
 914         if (r)
 915                 return r;
 916         /* Setup L2 cache */
 917         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 918                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 919                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 920         WREG32(VM_L2_CNTL2, 0);
 921         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 922         /* Setup TLB control */
 923         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 924                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 925                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 926                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 927         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 928         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 929         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 930         if (rdev->family == CHIP_RV740)
 931                 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
 932         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 933         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 934         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 935         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 936         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 937         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 938         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 939         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 940                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 941         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 942                         (u32)(rdev->dummy_page.addr >> 12));
 943         for (i = 1; i < 7; i++)
 944                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 945
 946         r600_pcie_gart_tlb_flush(rdev);
 947         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 948                  (unsigned)(rdev->mc.gtt_size >> 20),
 949                  (unsigned long long)rdev->gart.table_addr);
 950         rdev->gart.ready = true;
 951         return 0;
 952 }
 953
 954 static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 955 {
 956         u32 tmp;
 957         int i;
 958
 959         /* Disable all tables */
 960         for (i = 0; i < 7; i++)
 961                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 962
 963         /* Setup L2 cache */
 964         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 965                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 966         WREG32(VM_L2_CNTL2, 0);
 967         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 968         /* Setup TLB control */
 969         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 970         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 971         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 972         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 973         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 974         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 975         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 976         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 977         radeon_gart_table_vram_unpin(rdev);
 978 }
 979
 980 static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 981 {
 982         radeon_gart_fini(rdev);
 983         rv770_pcie_gart_disable(rdev);
 984         radeon_gart_table_vram_free(rdev);
 985 }
 986
 987
 988 static void rv770_agp_enable(struct radeon_device *rdev)
 989 {
 990         u32 tmp;
 991         int i;
 992
 993         /* Setup L2 cache */
 994         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 995                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 996                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 997         WREG32(VM_L2_CNTL2, 0);
 998         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 999         /* Setup TLB control */
1000         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1001                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1002                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
1003                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1004         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1005         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1006         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1007         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1008         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1009         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1010         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1011         for (i = 0; i < 7; i++)
1012                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1013 }
1014
1015 static void rv770_mc_program(struct radeon_device *rdev)
1016 {
1017         struct rv515_mc_save save;
1018         u32 tmp;
1019         int i, j;
1020
1021         /* Initialize HDP */
1022         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1023                 WREG32((0x2c14 + j), 0x00000000);
1024                 WREG32((0x2c18 + j), 0x00000000);
1025                 WREG32((0x2c1c + j), 0x00000000);
1026                 WREG32((0x2c20 + j), 0x00000000);
1027                 WREG32((0x2c24 + j), 0x00000000);
1028         }
1029         /* r7xx hw bug.  Read from HDP_DEBUG1 rather
1030          * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1031          */
1032         tmp = RREG32(HDP_DEBUG1);
1033
1034         rv515_mc_stop(rdev, &save);
1035         if (r600_mc_wait_for_idle(rdev)) {
1036                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1037         }
1038         /* Lockout access through VGA aperture*/
1039         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1040         /* Update configuration */
1041         if (rdev->flags & RADEON_IS_AGP) {
1042                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1043                         /* VRAM before AGP */
1044                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1045                                 rdev->mc.vram_start >> 12);
1046                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1047                                 rdev->mc.gtt_end >> 12);
1048                 } else {
1049                         /* VRAM after AGP */
1050                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1051                                 rdev->mc.gtt_start >> 12);
1052                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1053                                 rdev->mc.vram_end >> 12);
1054                 }
1055         } else {
1056                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1057                         rdev->mc.vram_start >> 12);
1058                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1059                         rdev->mc.vram_end >> 12);
1060         }
1061         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1062         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1063         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1064         WREG32(MC_VM_FB_LOCATION, tmp);
1065         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1066         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1067         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1068         if (rdev->flags & RADEON_IS_AGP) {
1069                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1070                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1071                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1072         } else {
1073                 WREG32(MC_VM_AGP_BASE, 0);
1074                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1075                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1076         }
1077         if (r600_mc_wait_for_idle(rdev)) {
1078                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1079         }
1080         rv515_mc_resume(rdev, &save);
1081         /* we need to own VRAM, so turn off the VGA renderer here
1082          * to stop it overwriting our objects */
1083         rv515_vga_render_disable(rdev);
1084 }
1085
1086
1087 /*
1088  * CP.
1089  */
1090 void r700_cp_stop(struct radeon_device *rdev)
1091 {
1092         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1093                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1094         WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1095         WREG32(SCRATCH_UMSK, 0);
1096         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1097 }
1098
1099 static int rv770_cp_load_microcode(struct radeon_device *rdev)
1100 {
1101         const __be32 *fw_data;
1102         int i;
1103
1104         if (!rdev->me_fw || !rdev->pfp_fw)
1105                 return -EINVAL;
1106
1107         r700_cp_stop(rdev);
1108         WREG32(CP_RB_CNTL,
1109 #ifdef __BIG_ENDIAN
1110                BUF_SWAP_32BIT |
1111 #endif
1112                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1113
1114         /* Reset cp */
1115         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1116         RREG32(GRBM_SOFT_RESET);
1117         mdelay(15);
1118         WREG32(GRBM_SOFT_RESET, 0);
1119
1120         fw_data = (const __be32 *)rdev->pfp_fw->data;
1121         WREG32(CP_PFP_UCODE_ADDR, 0);
1122         for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1123                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1124         WREG32(CP_PFP_UCODE_ADDR, 0);
1125
1126         fw_data = (const __be32 *)rdev->me_fw->data;
1127         WREG32(CP_ME_RAM_WADDR, 0);
1128         for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1129                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1130
1131         WREG32(CP_PFP_UCODE_ADDR, 0);
1132         WREG32(CP_ME_RAM_WADDR, 0);
1133         WREG32(CP_ME_RAM_RADDR, 0);
1134         return 0;
1135 }
1136
1137 void r700_cp_fini(struct radeon_device *rdev)
1138 {
1139         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1140         r700_cp_stop(rdev);
1141         radeon_ring_fini(rdev, ring);
1142         radeon_scratch_free(rdev, ring->rptr_save_reg);
1143 }
1144
1145 void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1146 {
1147         u32 tmp, i;
1148
1149         if (rdev->flags & RADEON_IS_IGP)
1150                 return;
1151
1152         tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1153         tmp &= SCLK_MUX_SEL_MASK;
1154         tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1155         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1156
1157         for (i = 0; i < rdev->usec_timeout; i++) {
1158                 if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1159                         break;
1160                 udelay(1);
1161         }
1162
1163         tmp &= ~SCLK_MUX_UPDATE;
1164         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1165
1166         tmp = RREG32(MPLL_CNTL_MODE);
1167         if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1168                 tmp &= ~RV730_MPLL_MCLK_SEL;
1169         else
1170                 tmp &= ~MPLL_MCLK_SEL;
1171         WREG32(MPLL_CNTL_MODE, tmp);
1172 }
1173
1174 /*
1175  * Core functions
1176  */
1177 static void rv770_gpu_init(struct radeon_device *rdev)
1178 {
1179         int i, j, num_qd_pipes;
1180         u32 ta_aux_cntl;
1181         u32 sx_debug_1;
1182         u32 smx_dc_ctl0;
1183         u32 db_debug3;
1184         u32 num_gs_verts_per_thread;
1185         u32 vgt_gs_per_es;
1186         u32 gs_prim_buffer_depth = 0;
1187         u32 sq_ms_fifo_sizes;
1188         u32 sq_config;
1189         u32 sq_thread_resource_mgmt;
1190         u32 hdp_host_path_cntl;
1191         u32 sq_dyn_gpr_size_simd_ab_0;
1192         u32 gb_tiling_config = 0;
1193         u32 cc_gc_shader_pipe_config = 0;
1194         u32 mc_arb_ramcfg;
1195         u32 db_debug4, tmp;
1196         u32 inactive_pipes, shader_pipe_config;
1197         u32 disabled_rb_mask;
1198         unsigned active_number;
1199
1200         /* setup chip specs */
1201         rdev->config.rv770.tiling_group_size = 256;
1202         switch (rdev->family) {
1203         case CHIP_RV770:
1204                 rdev->config.rv770.max_pipes = 4;
1205                 rdev->config.rv770.max_tile_pipes = 8;
1206                 rdev->config.rv770.max_simds = 10;
1207                 rdev->config.rv770.max_backends = 4;
1208                 rdev->config.rv770.max_gprs = 256;
1209                 rdev->config.rv770.max_threads = 248;
1210                 rdev->config.rv770.max_stack_entries = 512;
1211                 rdev->config.rv770.max_hw_contexts = 8;
1212                 rdev->config.rv770.max_gs_threads = 16 * 2;
1213                 rdev->config.rv770.sx_max_export_size = 128;
1214                 rdev->config.rv770.sx_max_export_pos_size = 16;
1215                 rdev->config.rv770.sx_max_export_smx_size = 112;
1216                 rdev->config.rv770.sq_num_cf_insts = 2;
1217
1218                 rdev->config.rv770.sx_num_of_sets = 7;
1219                 rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1220                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1221                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1222                 break;
1223         case CHIP_RV730:
1224                 rdev->config.rv770.max_pipes = 2;
1225                 rdev->config.rv770.max_tile_pipes = 4;
1226                 rdev->config.rv770.max_simds = 8;
1227                 rdev->config.rv770.max_backends = 2;
1228                 rdev->config.rv770.max_gprs = 128;
1229                 rdev->config.rv770.max_threads = 248;
1230                 rdev->config.rv770.max_stack_entries = 256;
1231                 rdev->config.rv770.max_hw_contexts = 8;
1232                 rdev->config.rv770.max_gs_threads = 16 * 2;
1233                 rdev->config.rv770.sx_max_export_size = 256;
1234                 rdev->config.rv770.sx_max_export_pos_size = 32;
1235                 rdev->config.rv770.sx_max_export_smx_size = 224;
1236                 rdev->config.rv770.sq_num_cf_insts = 2;
1237
1238                 rdev->config.rv770.sx_num_of_sets = 7;
1239                 rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1240                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1241                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1242                 if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1243                         rdev->config.rv770.sx_max_export_pos_size -= 16;
1244                         rdev->config.rv770.sx_max_export_smx_size += 16;
1245                 }
1246                 break;
1247         case CHIP_RV710:
1248                 rdev->config.rv770.max_pipes = 2;
1249                 rdev->config.rv770.max_tile_pipes = 2;
1250                 rdev->config.rv770.max_simds = 2;
1251                 rdev->config.rv770.max_backends = 1;
1252                 rdev->config.rv770.max_gprs = 256;
1253                 rdev->config.rv770.max_threads = 192;
1254                 rdev->config.rv770.max_stack_entries = 256;
1255                 rdev->config.rv770.max_hw_contexts = 4;
1256                 rdev->config.rv770.max_gs_threads = 8 * 2;
1257                 rdev->config.rv770.sx_max_export_size = 128;
1258                 rdev->config.rv770.sx_max_export_pos_size = 16;
1259                 rdev->config.rv770.sx_max_export_smx_size = 112;
1260                 rdev->config.rv770.sq_num_cf_insts = 1;
1261
1262                 rdev->config.rv770.sx_num_of_sets = 7;
1263                 rdev->config.rv770.sc_prim_fifo_size = 0x40;
1264                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1265                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1266                 break;
1267         case CHIP_RV740:
1268                 rdev->config.rv770.max_pipes = 4;
1269                 rdev->config.rv770.max_tile_pipes = 4;
1270                 rdev->config.rv770.max_simds = 8;
1271                 rdev->config.rv770.max_backends = 4;
1272                 rdev->config.rv770.max_gprs = 256;
1273                 rdev->config.rv770.max_threads = 248;
1274                 rdev->config.rv770.max_stack_entries = 512;
1275                 rdev->config.rv770.max_hw_contexts = 8;
1276                 rdev->config.rv770.max_gs_threads = 16 * 2;
1277                 rdev->config.rv770.sx_max_export_size = 256;
1278                 rdev->config.rv770.sx_max_export_pos_size = 32;
1279                 rdev->config.rv770.sx_max_export_smx_size = 224;
1280                 rdev->config.rv770.sq_num_cf_insts = 2;
1281
1282                 rdev->config.rv770.sx_num_of_sets = 7;
1283                 rdev->config.rv770.sc_prim_fifo_size = 0x100;
1284                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1285                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1286
1287                 if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1288                         rdev->config.rv770.sx_max_export_pos_size -= 16;
1289                         rdev->config.rv770.sx_max_export_smx_size += 16;
1290                 }
1291                 break;
1292         default:
1293                 break;
1294         }
1295
1296         /* Initialize HDP */
1297         j = 0;
1298         for (i = 0; i < 32; i++) {
1299                 WREG32((0x2c14 + j), 0x00000000);
1300                 WREG32((0x2c18 + j), 0x00000000);
1301                 WREG32((0x2c1c + j), 0x00000000);
1302                 WREG32((0x2c20 + j), 0x00000000);
1303                 WREG32((0x2c24 + j), 0x00000000);
1304                 j += 0x18;
1305         }
1306
1307         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1308
1309         /* setup tiling, simd, pipe config */
1310         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1311
1312         shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1313         inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1314         for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1315                 if (!(inactive_pipes & tmp)) {
1316                         active_number++;
1317                 }
1318                 tmp <<= 1;
1319         }
1320         if (active_number == 1) {
1321                 WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1322         } else {
1323                 WREG32(SPI_CONFIG_CNTL, 0);
1324         }
1325
1326         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1327         tmp = rdev->config.rv770.max_simds -
1328                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1329         rdev->config.rv770.active_simds = tmp;
1330
1331         switch (rdev->config.rv770.max_tile_pipes) {
1332         case 1:
1333         default:
1334                 gb_tiling_config = PIPE_TILING(0);
1335                 break;
1336         case 2:
1337                 gb_tiling_config = PIPE_TILING(1);
1338                 break;
1339         case 4:
1340                 gb_tiling_config = PIPE_TILING(2);
1341                 break;
1342         case 8:
1343                 gb_tiling_config = PIPE_TILING(3);
1344                 break;
1345         }
1346         rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1347
1348         disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1349         tmp = 0;
1350         for (i = 0; i < rdev->config.rv770.max_backends; i++)
1351                 tmp |= (1 << i);
1352         /* if all the backends are disabled, fix it up here */
1353         if ((disabled_rb_mask & tmp) == tmp) {
1354                 for (i = 0; i < rdev->config.rv770.max_backends; i++)
1355                         disabled_rb_mask &= ~(1 << i);
1356         }
1357         tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1358         tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1359                                         R7XX_MAX_BACKENDS, disabled_rb_mask);
1360         gb_tiling_config |= tmp << 16;
1361         rdev->config.rv770.backend_map = tmp;
1362
1363         if (rdev->family == CHIP_RV770)
1364                 gb_tiling_config |= BANK_TILING(1);
1365         else {
1366                 if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1367                         gb_tiling_config |= BANK_TILING(1);
1368                 else
1369                         gb_tiling_config |= BANK_TILING(0);
1370         }
1371         rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1372         gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1373         if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1374                 gb_tiling_config |= ROW_TILING(3);
1375                 gb_tiling_config |= SAMPLE_SPLIT(3);
1376         } else {
1377                 gb_tiling_config |=
1378                         ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1379                 gb_tiling_config |=
1380                         SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1381         }
1382
1383         gb_tiling_config |= BANK_SWAPS(1);
1384         rdev->config.rv770.tile_config = gb_tiling_config;
1385
1386         WREG32(GB_TILING_CONFIG, gb_tiling_config);
1387         WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1388         WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1389         WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1390         WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1391         if (rdev->family == CHIP_RV730) {
1392                 WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1393                 WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1394                 WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1395         }
1396
1397         WREG32(CGTS_SYS_TCC_DISABLE, 0);
1398         WREG32(CGTS_TCC_DISABLE, 0);
1399         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1400         WREG32(CGTS_USER_TCC_DISABLE, 0);
1401
1402
1403         num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1404         WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1405         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1406
1407         /* set HW defaults for 3D engine */
1408         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1409                                      ROQ_IB2_START(0x2b)));
1410
1411         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1412
1413         ta_aux_cntl = RREG32(TA_CNTL_AUX);
1414         WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1415
1416         sx_debug_1 = RREG32(SX_DEBUG_1);
1417         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1418         WREG32(SX_DEBUG_1, sx_debug_1);
1419
1420         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1421         smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1422         smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1423         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1424
1425         if (rdev->family != CHIP_RV740)
1426                 WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1427                                        GS_FLUSH_CTL(4) |
1428                                        ACK_FLUSH_CTL(3) |
1429                                        SYNC_FLUSH_CTL));
1430
1431         if (rdev->family != CHIP_RV770)
1432                 WREG32(SMX_SAR_CTL0, 0x00003f3f);
1433
1434         db_debug3 = RREG32(DB_DEBUG3);
1435         db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1436         switch (rdev->family) {
1437         case CHIP_RV770:
1438         case CHIP_RV740:
1439                 db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1440                 break;
1441         case CHIP_RV710:
1442         case CHIP_RV730:
1443         default:
1444                 db_debug3 |= DB_CLK_OFF_DELAY(2);
1445                 break;
1446         }
1447         WREG32(DB_DEBUG3, db_debug3);
1448
1449         if (rdev->family != CHIP_RV770) {
1450                 db_debug4 = RREG32(DB_DEBUG4);
1451                 db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1452                 WREG32(DB_DEBUG4, db_debug4);
1453         }
1454
1455         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1456                                         POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1457                                         SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1458
1459         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1460                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1461                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1462
1463         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1464
1465         WREG32(VGT_NUM_INSTANCES, 1);
1466
1467         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1468
1469         WREG32(CP_PERFMON_CNTL, 0);
1470
1471         sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1472                             DONE_FIFO_HIWATER(0xe0) |
1473                             ALU_UPDATE_FIFO_HIWATER(0x8));
1474         switch (rdev->family) {
1475         case CHIP_RV770:
1476         case CHIP_RV730:
1477         case CHIP_RV710:
1478                 sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1479                 break;
1480         case CHIP_RV740:
1481         default:
1482                 sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1483                 break;
1484         }
1485         WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1486
1487         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1488          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1489          */
1490         sq_config = RREG32(SQ_CONFIG);
1491         sq_config &= ~(PS_PRIO(3) |
1492                        VS_PRIO(3) |
1493                        GS_PRIO(3) |
1494                        ES_PRIO(3));
1495         sq_config |= (DX9_CONSTS |
1496                       VC_ENABLE |
1497                       EXPORT_SRC_C |
1498                       PS_PRIO(0) |
1499                       VS_PRIO(1) |
1500                       GS_PRIO(2) |
1501                       ES_PRIO(3));
1502         if (rdev->family == CHIP_RV710)
1503                 /* no vertex cache */
1504                 sq_config &= ~VC_ENABLE;
1505
1506         WREG32(SQ_CONFIG, sq_config);
1507
1508         WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1509                                          NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1510                                          NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1511
1512         WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1513                                          NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1514
1515         sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1516                                    NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1517                                    NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1518         if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1519                 sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1520         else
1521                 sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1522         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1523
1524         WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1525                                                      NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1526
1527         WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1528                                                      NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1529
1530         sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1531                                      SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1532                                      SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1533                                      SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1534
1535         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1536         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1537         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1538         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1539         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1540         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1541         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1542         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1543
1544         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1545                                           FORCE_EOV_MAX_REZ_CNT(255)));
1546
1547         if (rdev->family == CHIP_RV710)
1548                 WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1549                                                 AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1550         else
1551                 WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1552                                                 AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1553
1554         switch (rdev->family) {
1555         case CHIP_RV770:
1556         case CHIP_RV730:
1557         case CHIP_RV740:
1558                 gs_prim_buffer_depth = 384;
1559                 break;
1560         case CHIP_RV710:
1561                 gs_prim_buffer_depth = 128;
1562                 break;
1563         default:
1564                 break;
1565         }
1566
1567         num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1568         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1569         /* Max value for this is 256 */
1570         if (vgt_gs_per_es > 256)
1571                 vgt_gs_per_es = 256;
1572
1573         WREG32(VGT_ES_PER_GS, 128);
1574         WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1575         WREG32(VGT_GS_PER_VS, 2);
1576
1577         /* more default values. 2D/3D driver should adjust as needed */
1578         WREG32(VGT_GS_VERTEX_REUSE, 16);
1579         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1580         WREG32(VGT_STRMOUT_EN, 0);
1581         WREG32(SX_MISC, 0);
1582         WREG32(PA_SC_MODE_CNTL, 0);
1583         WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1584         WREG32(PA_SC_AA_CONFIG, 0);
1585         WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1586         WREG32(PA_SC_LINE_STIPPLE, 0);
1587         WREG32(SPI_INPUT_Z, 0);
1588         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1589         WREG32(CB_COLOR7_FRAG, 0);
1590
1591         /* clear render buffer base addresses */
1592         WREG32(CB_COLOR0_BASE, 0);
1593         WREG32(CB_COLOR1_BASE, 0);
1594         WREG32(CB_COLOR2_BASE, 0);
1595         WREG32(CB_COLOR3_BASE, 0);
1596         WREG32(CB_COLOR4_BASE, 0);
1597         WREG32(CB_COLOR5_BASE, 0);
1598         WREG32(CB_COLOR6_BASE, 0);
1599         WREG32(CB_COLOR7_BASE, 0);
1600
1601         WREG32(TCP_CNTL, 0);
1602
1603         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1604         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1605
1606         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1607
1608         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1609                                           NUM_CLIP_SEQ(3)));
1610         WREG32(VC_ENHANCE, 0);
1611 }
1612
1613 void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1614 {
1615         u64 size_bf, size_af;
1616
1617         if (mc->mc_vram_size > 0xE0000000) {
1618                 /* leave room for at least 512M GTT */
1619                 dev_warn(rdev->dev, "limiting VRAM\n");
1620                 mc->real_vram_size = 0xE0000000;
1621                 mc->mc_vram_size = 0xE0000000;
1622         }
1623         if (rdev->flags & RADEON_IS_AGP) {
1624                 size_bf = mc->gtt_start;
1625                 size_af = mc->mc_mask - mc->gtt_end;
1626                 if (size_bf > size_af) {
1627                         if (mc->mc_vram_size > size_bf) {
1628                                 dev_warn(rdev->dev, "limiting VRAM\n");
1629                                 mc->real_vram_size = size_bf;
1630                                 mc->mc_vram_size = size_bf;
1631                         }
1632                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1633                 } else {
1634                         if (mc->mc_vram_size > size_af) {
1635                                 dev_warn(rdev->dev, "limiting VRAM\n");
1636                                 mc->real_vram_size = size_af;
1637                                 mc->mc_vram_size = size_af;
1638                         }
1639                         mc->vram_start = mc->gtt_end + 1;
1640                 }
1641                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1642                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1643                                 mc->mc_vram_size >> 20, mc->vram_start,
1644                                 mc->vram_end, mc->real_vram_size >> 20);
1645         } else {
1646                 radeon_vram_location(rdev, &rdev->mc, 0);
1647                 rdev->mc.gtt_base_align = 0;
1648                 radeon_gtt_location(rdev, mc);
1649         }
1650 }
1651
1652 static int rv770_mc_init(struct radeon_device *rdev)
1653 {
1654         u32 tmp;
1655         int chansize, numchan;
1656
1657         /* Get VRAM informations */
1658         rdev->mc.vram_is_ddr = true;
1659         tmp = RREG32(MC_ARB_RAMCFG);
1660         if (tmp & CHANSIZE_OVERRIDE) {
1661                 chansize = 16;
1662         } else if (tmp & CHANSIZE_MASK) {
1663                 chansize = 64;
1664         } else {
1665                 chansize = 32;
1666         }
1667         tmp = RREG32(MC_SHARED_CHMAP);
1668         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1669         case 0:
1670         default:
1671                 numchan = 1;
1672                 break;
1673         case 1:
1674                 numchan = 2;
1675                 break;
1676         case 2:
1677                 numchan = 4;
1678                 break;
1679         case 3:
1680                 numchan = 8;
1681                 break;
1682         }
1683         rdev->mc.vram_width = numchan * chansize;
1684         /* Could aper size report 0 ? */
1685         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1686         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1687         /* Setup GPU memory space */
1688         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1689         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1690         rdev->mc.visible_vram_size = rdev->mc.aper_size;
1691         r700_vram_gtt_location(rdev, &rdev->mc);
1692         radeon_update_bandwidth_info(rdev);
1693
1694         return 0;
1695 }
1696
1697 static void rv770_uvd_init(struct radeon_device *rdev)
1698 {
1699         int r;
1700
1701         if (!rdev->has_uvd)
1702                 return;
1703
1704         r = radeon_uvd_init(rdev);
1705         if (r) {
1706                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1707                 /*
1708                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1709                  * to early fails uvd_v2_2_resume() and thus nothing happens
1710                  * there. So it is pointless to try to go through that code
1711                  * hence why we disable uvd here.
1712                  */
1713                 rdev->has_uvd = false;
1714                 return;
1715         }
1716         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1717         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
1718 }
1719
1720 static void rv770_uvd_start(struct radeon_device *rdev)
1721 {
1722         int r;
1723
1724         if (!rdev->has_uvd)
1725                 return;
1726
1727         r = uvd_v2_2_resume(rdev);
1728         if (r) {
1729                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
1730                 goto error;
1731         }
1732         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
1733         if (r) {
1734                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
1735                 goto error;
1736         }
1737         return;
1738
1739 error:
1740         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1741 }
1742
1743 static void rv770_uvd_resume(struct radeon_device *rdev)
1744 {
1745         struct radeon_ring *ring;
1746         int r;
1747
1748         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
1749                 return;
1750
1751         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1752         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
1753         if (r) {
1754                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
1755                 return;
1756         }
1757         r = uvd_v1_0_init(rdev);
1758         if (r) {
1759                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
1760                 return;
1761         }
1762 }
1763
1764 static int rv770_startup(struct radeon_device *rdev)
1765 {
1766         struct radeon_ring *ring;
1767         int r;
1768
1769         /* enable pcie gen2 link */
1770         rv770_pcie_gen2_enable(rdev);
1771
1772         /* scratch needs to be initialized before MC */
1773         r = r600_vram_scratch_init(rdev);
1774         if (r)
1775                 return r;
1776
1777         rv770_mc_program(rdev);
1778
1779         if (rdev->flags & RADEON_IS_AGP) {
1780                 rv770_agp_enable(rdev);
1781         } else {
1782                 r = rv770_pcie_gart_enable(rdev);
1783                 if (r)
1784                         return r;
1785         }
1786
1787         rv770_gpu_init(rdev);
1788
1789         /* allocate wb buffer */
1790         r = radeon_wb_init(rdev);
1791         if (r)
1792                 return r;
1793
1794         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1795         if (r) {
1796                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1797                 return r;
1798         }
1799
1800         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1801         if (r) {
1802                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1803                 return r;
1804         }
1805
1806         rv770_uvd_start(rdev);
1807
1808         /* Enable IRQ */
1809         if (!rdev->irq.installed) {
1810                 r = radeon_irq_kms_init(rdev);
1811                 if (r)
1812                         return r;
1813         }
1814
1815         r = r600_irq_init(rdev);
1816         if (r) {
1817                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1818                 radeon_irq_kms_fini(rdev);
1819                 return r;
1820         }
1821         r600_irq_set(rdev);
1822
1823         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1824         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1825                              RADEON_CP_PACKET2);
1826         if (r)
1827                 return r;
1828
1829         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1830         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1831                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1832         if (r)
1833                 return r;
1834
1835         r = rv770_cp_load_microcode(rdev);
1836         if (r)
1837                 return r;
1838         r = r600_cp_resume(rdev);
1839         if (r)
1840                 return r;
1841
1842         r = r600_dma_resume(rdev);
1843         if (r)
1844                 return r;
1845
1846         rv770_uvd_resume(rdev);
1847
1848         r = radeon_ib_pool_init(rdev);
1849         if (r) {
1850                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1851                 return r;
1852         }
1853
1854         r = radeon_audio_init(rdev);
1855         if (r) {
1856                 DRM_ERROR("radeon: audio init failed\n");
1857                 return r;
1858         }
1859
1860         return 0;
1861 }
1862
1863 int rv770_resume(struct radeon_device *rdev)
1864 {
1865         int r;
1866
1867         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1868          * posting will perform necessary task to bring back GPU into good
1869          * shape.
1870          */
1871         /* post card */
1872         atom_asic_init(rdev->mode_info.atom_context);
1873
1874         /* init golden registers */
1875         rv770_init_golden_registers(rdev);
1876
1877         if (rdev->pm.pm_method == PM_METHOD_DPM)
1878                 radeon_pm_resume(rdev);
1879
1880         rdev->accel_working = true;
1881         r = rv770_startup(rdev);
1882         if (r) {
1883                 DRM_ERROR("r600 startup failed on resume\n");
1884                 rdev->accel_working = false;
1885                 return r;
1886         }
1887
1888         return r;
1889
1890 }
1891
1892 int rv770_suspend(struct radeon_device *rdev)
1893 {
1894         radeon_pm_suspend(rdev);
1895         radeon_audio_fini(rdev);
1896         if (rdev->has_uvd) {
1897                 radeon_uvd_suspend(rdev);
1898                 uvd_v1_0_fini(rdev);
1899         }
1900         r700_cp_stop(rdev);
1901         r600_dma_stop(rdev);
1902         r600_irq_suspend(rdev);
1903         radeon_wb_disable(rdev);
1904         rv770_pcie_gart_disable(rdev);
1905
1906         return 0;
1907 }
1908
1909 /* Plan is to move initialization in that function and use
1910  * helper function so that radeon_device_init pretty much
1911  * do nothing more than calling asic specific function. This
1912  * should also allow to remove a bunch of callback function
1913  * like vram_info.
1914  */
1915 int rv770_init(struct radeon_device *rdev)
1916 {
1917         int r;
1918
1919         /* Read BIOS */
1920         if (!radeon_get_bios(rdev)) {
1921                 if (ASIC_IS_AVIVO(rdev))
1922                         return -EINVAL;
1923         }
1924         /* Must be an ATOMBIOS */
1925         if (!rdev->is_atom_bios) {
1926                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1927                 return -EINVAL;
1928         }
1929         r = radeon_atombios_init(rdev);
1930         if (r)
1931                 return r;
1932         /* Post card if necessary */
1933         if (!radeon_card_posted(rdev)) {
1934                 if (!rdev->bios) {
1935                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1936                         return -EINVAL;
1937                 }
1938                 DRM_INFO("GPU not posted. posting now...\n");
1939                 atom_asic_init(rdev->mode_info.atom_context);
1940         }
1941         /* init golden registers */
1942         rv770_init_golden_registers(rdev);
1943         /* Initialize scratch registers */
1944         r600_scratch_init(rdev);
1945         /* Initialize surface registers */
1946         radeon_surface_init(rdev);
1947         /* Initialize clocks */
1948         radeon_get_clock_info(rdev->ddev);
1949         /* Fence driver */
1950         radeon_fence_driver_init(rdev);
1951         /* initialize AGP */
1952         if (rdev->flags & RADEON_IS_AGP) {
1953                 r = radeon_agp_init(rdev);
1954                 if (r)
1955                         radeon_agp_disable(rdev);
1956         }
1957         r = rv770_mc_init(rdev);
1958         if (r)
1959                 return r;
1960         /* Memory manager */
1961         r = radeon_bo_init(rdev);
1962         if (r)
1963                 return r;
1964
1965         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1966                 r = r600_init_microcode(rdev);
1967                 if (r) {
1968                         DRM_ERROR("Failed to load firmware!\n");
1969                         /*(DEBLOBBED)*/
1970                 }
1971         }
1972
1973         /* Initialize power management */
1974         radeon_pm_init(rdev);
1975
1976         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1977         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1978
1979         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1980         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1981
1982         rv770_uvd_init(rdev);
1983
1984         rdev->ih.ring_obj = NULL;
1985         r600_ih_ring_init(rdev, 64 * 1024);
1986
1987         r = r600_pcie_gart_init(rdev);
1988         if (r)
1989                 return r;
1990
1991         rdev->accel_working = true;
1992         r = rv770_startup(rdev);
1993         if (r) {
1994                 dev_err(rdev->dev, "disabling GPU acceleration\n");
1995                 r700_cp_fini(rdev);
1996                 r600_dma_fini(rdev);
1997                 r600_irq_fini(rdev);
1998                 radeon_wb_fini(rdev);
1999                 radeon_ib_pool_fini(rdev);
2000                 radeon_irq_kms_fini(rdev);
2001                 rv770_pcie_gart_fini(rdev);
2002                 rdev->accel_working = false;
2003         }
2004
2005         return 0;
2006 }
2007
2008 void rv770_fini(struct radeon_device *rdev)
2009 {
2010         radeon_pm_fini(rdev);
2011         r700_cp_fini(rdev);
2012         r600_dma_fini(rdev);
2013         r600_irq_fini(rdev);
2014         radeon_wb_fini(rdev);
2015         radeon_ib_pool_fini(rdev);
2016         radeon_irq_kms_fini(rdev);
2017         uvd_v1_0_fini(rdev);
2018         radeon_uvd_fini(rdev);
2019         rv770_pcie_gart_fini(rdev);
2020         r600_vram_scratch_fini(rdev);
2021         radeon_gem_fini(rdev);
2022         radeon_fence_driver_fini(rdev);
2023         radeon_agp_fini(rdev);
2024         radeon_bo_fini(rdev);
2025         radeon_atombios_fini(rdev);
2026         kfree(rdev->bios);
2027         rdev->bios = NULL;
2028 }
2029
2030 static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
2031 {
2032         u32 link_width_cntl, lanes, speed_cntl, tmp;
2033         u16 link_cntl2;
2034
2035         if (radeon_pcie_gen2 == 0)
2036                 return;
2037
2038         if (rdev->flags & RADEON_IS_IGP)
2039                 return;
2040
2041         if (!(rdev->flags & RADEON_IS_PCIE))
2042                 return;
2043
2044         /* x2 cards have a special sequence */
2045         if (ASIC_IS_X2(rdev))
2046                 return;
2047
2048         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
2049                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2050                 return;
2051
2052         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
2053
2054         /* advertise upconfig capability */
2055         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2056         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2057         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2058         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2059         if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2060                 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2061                 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2062                                      LC_RECONFIG_ARC_MISSING_ESCAPE);
2063                 link_width_cntl |= lanes | LC_RECONFIG_NOW |
2064                         LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2065                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2066         } else {
2067                 link_width_cntl |= LC_UPCONFIGURE_DIS;
2068                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2069         }
2070
2071         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2072         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2073             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2074
2075                 tmp = RREG32(0x541c);
2076                 WREG32(0x541c, tmp | 0x8);
2077                 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2078                 link_cntl2 = RREG16(0x4088);
2079                 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2080                 link_cntl2 |= 0x2;
2081                 WREG16(0x4088, link_cntl2);
2082                 WREG32(MM_CFGREGS_CNTL, 0);
2083
2084                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2085                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2086                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2087
2088                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2089                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2090                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2091
2092                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2093                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2094                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2095
2096                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2097                 speed_cntl |= LC_GEN2_EN_STRAP;
2098                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2099
2100         } else {
2101                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2102                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2103                 if (1)
2104                         link_width_cntl |= LC_UPCONFIGURE_DIS;
2105                 else
2106                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2107                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2108         }
2109 }