drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/math64.h>
  25 #include <linux/pci.h>
  26 #include <linux/seq_file.h>
  27
  28 #include "atom.h"
  29 #include "ni_dpm.h"
  30 #include "nid.h"
  31 #include "r600_dpm.h"
  32 #include "radeon.h"
  33 #include "radeon_asic.h"
  34
  35 #define MC_CG_ARB_FREQ_F0           0x0a
  36 #define MC_CG_ARB_FREQ_F1           0x0b
  37 #define MC_CG_ARB_FREQ_F2           0x0c
  38 #define MC_CG_ARB_FREQ_F3           0x0d
  39
  40 #define SMC_RAM_END 0xC000
  41
  42 static const struct ni_cac_weights cac_weights_cayman_xt =
  43 {
  44         0x15,
  45         0x2,
  46         0x19,
  47         0x2,
  48         0x8,
  49         0x14,
  50         0x2,
  51         0x16,
  52         0xE,
  53         0x17,
  54         0x13,
  55         0x2B,
  56         0x10,
  57         0x7,
  58         0x5,
  59         0x5,
  60         0x5,
  61         0x2,
  62         0x3,
  63         0x9,
  64         0x10,
  65         0x10,
  66         0x2B,
  67         0xA,
  68         0x9,
  69         0x4,
  70         0xD,
  71         0xD,
  72         0x3E,
  73         0x18,
  74         0x14,
  75         0,
  76         0x3,
  77         0x3,
  78         0x5,
  79         0,
  80         0x2,
  81         0,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0,
  89         0,
  90         0x1CC,
  91         0,
  92         0x164,
  93         1,
  94         1,
  95         1,
  96         1,
  97         12,
  98         12,
  99         12,
 100         0x12,
 101         0x1F,
 102         132,
 103         5,
 104         7,
 105         0,
 106         { 0, 0, 0, 0, 0, 0, 0, 0 },
 107         { 0, 0, 0, 0 },
 108         true
 109 };
 110
 111 static const struct ni_cac_weights cac_weights_cayman_pro =
 112 {
 113         0x16,
 114         0x4,
 115         0x10,
 116         0x2,
 117         0xA,
 118         0x16,
 119         0x2,
 120         0x18,
 121         0x10,
 122         0x1A,
 123         0x16,
 124         0x2D,
 125         0x12,
 126         0xA,
 127         0x6,
 128         0x6,
 129         0x6,
 130         0x2,
 131         0x4,
 132         0xB,
 133         0x11,
 134         0x11,
 135         0x2D,
 136         0xC,
 137         0xC,
 138         0x7,
 139         0x10,
 140         0x10,
 141         0x3F,
 142         0x1A,
 143         0x16,
 144         0,
 145         0x7,
 146         0x4,
 147         0x6,
 148         1,
 149         0x2,
 150         0x1,
 151         0,
 152         0,
 153         0,
 154         0,
 155         0,
 156         0,
 157         0x30,
 158         0,
 159         0x1CF,
 160         0,
 161         0x166,
 162         1,
 163         1,
 164         1,
 165         1,
 166         12,
 167         12,
 168         12,
 169         0x15,
 170         0x1F,
 171         132,
 172         6,
 173         6,
 174         0,
 175         { 0, 0, 0, 0, 0, 0, 0, 0 },
 176         { 0, 0, 0, 0 },
 177         true
 178 };
 179
 180 static const struct ni_cac_weights cac_weights_cayman_le =
 181 {
 182         0x7,
 183         0xE,
 184         0x1,
 185         0xA,
 186         0x1,
 187         0x3F,
 188         0x2,
 189         0x18,
 190         0x10,
 191         0x1A,
 192         0x1,
 193         0x3F,
 194         0x1,
 195         0xE,
 196         0x6,
 197         0x6,
 198         0x6,
 199         0x2,
 200         0x4,
 201         0x9,
 202         0x1A,
 203         0x1A,
 204         0x2C,
 205         0xA,
 206         0x11,
 207         0x8,
 208         0x19,
 209         0x19,
 210         0x1,
 211         0x1,
 212         0x1A,
 213         0,
 214         0x8,
 215         0x5,
 216         0x8,
 217         0x1,
 218         0x3,
 219         0x1,
 220         0,
 221         0,
 222         0,
 223         0,
 224         0,
 225         0,
 226         0x38,
 227         0x38,
 228         0x239,
 229         0x3,
 230         0x18A,
 231         1,
 232         1,
 233         1,
 234         1,
 235         12,
 236         12,
 237         12,
 238         0x15,
 239         0x22,
 240         132,
 241         6,
 242         6,
 243         0,
 244         { 0, 0, 0, 0, 0, 0, 0, 0 },
 245         { 0, 0, 0, 0 },
 246         true
 247 };
 248
 249 #define NISLANDS_MGCG_SEQUENCE  300
 250
 251 static const u32 cayman_cgcg_cgls_default[] =
 252 {
 253         0x000008f8, 0x00000010, 0xffffffff,
 254         0x000008fc, 0x00000000, 0xffffffff,
 255         0x000008f8, 0x00000011, 0xffffffff,
 256         0x000008fc, 0x00000000, 0xffffffff,
 257         0x000008f8, 0x00000012, 0xffffffff,
 258         0x000008fc, 0x00000000, 0xffffffff,
 259         0x000008f8, 0x00000013, 0xffffffff,
 260         0x000008fc, 0x00000000, 0xffffffff,
 261         0x000008f8, 0x00000014, 0xffffffff,
 262         0x000008fc, 0x00000000, 0xffffffff,
 263         0x000008f8, 0x00000015, 0xffffffff,
 264         0x000008fc, 0x00000000, 0xffffffff,
 265         0x000008f8, 0x00000016, 0xffffffff,
 266         0x000008fc, 0x00000000, 0xffffffff,
 267         0x000008f8, 0x00000017, 0xffffffff,
 268         0x000008fc, 0x00000000, 0xffffffff,
 269         0x000008f8, 0x00000018, 0xffffffff,
 270         0x000008fc, 0x00000000, 0xffffffff,
 271         0x000008f8, 0x00000019, 0xffffffff,
 272         0x000008fc, 0x00000000, 0xffffffff,
 273         0x000008f8, 0x0000001a, 0xffffffff,
 274         0x000008fc, 0x00000000, 0xffffffff,
 275         0x000008f8, 0x0000001b, 0xffffffff,
 276         0x000008fc, 0x00000000, 0xffffffff,
 277         0x000008f8, 0x00000020, 0xffffffff,
 278         0x000008fc, 0x00000000, 0xffffffff,
 279         0x000008f8, 0x00000021, 0xffffffff,
 280         0x000008fc, 0x00000000, 0xffffffff,
 281         0x000008f8, 0x00000022, 0xffffffff,
 282         0x000008fc, 0x00000000, 0xffffffff,
 283         0x000008f8, 0x00000023, 0xffffffff,
 284         0x000008fc, 0x00000000, 0xffffffff,
 285         0x000008f8, 0x00000024, 0xffffffff,
 286         0x000008fc, 0x00000000, 0xffffffff,
 287         0x000008f8, 0x00000025, 0xffffffff,
 288         0x000008fc, 0x00000000, 0xffffffff,
 289         0x000008f8, 0x00000026, 0xffffffff,
 290         0x000008fc, 0x00000000, 0xffffffff,
 291         0x000008f8, 0x00000027, 0xffffffff,
 292         0x000008fc, 0x00000000, 0xffffffff,
 293         0x000008f8, 0x00000028, 0xffffffff,
 294         0x000008fc, 0x00000000, 0xffffffff,
 295         0x000008f8, 0x00000029, 0xffffffff,
 296         0x000008fc, 0x00000000, 0xffffffff,
 297         0x000008f8, 0x0000002a, 0xffffffff,
 298         0x000008fc, 0x00000000, 0xffffffff,
 299         0x000008f8, 0x0000002b, 0xffffffff,
 300         0x000008fc, 0x00000000, 0xffffffff
 301 };
 302 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 303
 304 static const u32 cayman_cgcg_cgls_disable[] =
 305 {
 306         0x000008f8, 0x00000010, 0xffffffff,
 307         0x000008fc, 0xffffffff, 0xffffffff,
 308         0x000008f8, 0x00000011, 0xffffffff,
 309         0x000008fc, 0xffffffff, 0xffffffff,
 310         0x000008f8, 0x00000012, 0xffffffff,
 311         0x000008fc, 0xffffffff, 0xffffffff,
 312         0x000008f8, 0x00000013, 0xffffffff,
 313         0x000008fc, 0xffffffff, 0xffffffff,
 314         0x000008f8, 0x00000014, 0xffffffff,
 315         0x000008fc, 0xffffffff, 0xffffffff,
 316         0x000008f8, 0x00000015, 0xffffffff,
 317         0x000008fc, 0xffffffff, 0xffffffff,
 318         0x000008f8, 0x00000016, 0xffffffff,
 319         0x000008fc, 0xffffffff, 0xffffffff,
 320         0x000008f8, 0x00000017, 0xffffffff,
 321         0x000008fc, 0xffffffff, 0xffffffff,
 322         0x000008f8, 0x00000018, 0xffffffff,
 323         0x000008fc, 0xffffffff, 0xffffffff,
 324         0x000008f8, 0x00000019, 0xffffffff,
 325         0x000008fc, 0xffffffff, 0xffffffff,
 326         0x000008f8, 0x0000001a, 0xffffffff,
 327         0x000008fc, 0xffffffff, 0xffffffff,
 328         0x000008f8, 0x0000001b, 0xffffffff,
 329         0x000008fc, 0xffffffff, 0xffffffff,
 330         0x000008f8, 0x00000020, 0xffffffff,
 331         0x000008fc, 0x00000000, 0xffffffff,
 332         0x000008f8, 0x00000021, 0xffffffff,
 333         0x000008fc, 0x00000000, 0xffffffff,
 334         0x000008f8, 0x00000022, 0xffffffff,
 335         0x000008fc, 0x00000000, 0xffffffff,
 336         0x000008f8, 0x00000023, 0xffffffff,
 337         0x000008fc, 0x00000000, 0xffffffff,
 338         0x000008f8, 0x00000024, 0xffffffff,
 339         0x000008fc, 0x00000000, 0xffffffff,
 340         0x000008f8, 0x00000025, 0xffffffff,
 341         0x000008fc, 0x00000000, 0xffffffff,
 342         0x000008f8, 0x00000026, 0xffffffff,
 343         0x000008fc, 0x00000000, 0xffffffff,
 344         0x000008f8, 0x00000027, 0xffffffff,
 345         0x000008fc, 0x00000000, 0xffffffff,
 346         0x000008f8, 0x00000028, 0xffffffff,
 347         0x000008fc, 0x00000000, 0xffffffff,
 348         0x000008f8, 0x00000029, 0xffffffff,
 349         0x000008fc, 0x00000000, 0xffffffff,
 350         0x000008f8, 0x0000002a, 0xffffffff,
 351         0x000008fc, 0x00000000, 0xffffffff,
 352         0x000008f8, 0x0000002b, 0xffffffff,
 353         0x000008fc, 0x00000000, 0xffffffff,
 354         0x00000644, 0x000f7902, 0x001f4180,
 355         0x00000644, 0x000f3802, 0x001f4180
 356 };
 357 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 358
 359 static const u32 cayman_cgcg_cgls_enable[] =
 360 {
 361         0x00000644, 0x000f7882, 0x001f4080,
 362         0x000008f8, 0x00000010, 0xffffffff,
 363         0x000008fc, 0x00000000, 0xffffffff,
 364         0x000008f8, 0x00000011, 0xffffffff,
 365         0x000008fc, 0x00000000, 0xffffffff,
 366         0x000008f8, 0x00000012, 0xffffffff,
 367         0x000008fc, 0x00000000, 0xffffffff,
 368         0x000008f8, 0x00000013, 0xffffffff,
 369         0x000008fc, 0x00000000, 0xffffffff,
 370         0x000008f8, 0x00000014, 0xffffffff,
 371         0x000008fc, 0x00000000, 0xffffffff,
 372         0x000008f8, 0x00000015, 0xffffffff,
 373         0x000008fc, 0x00000000, 0xffffffff,
 374         0x000008f8, 0x00000016, 0xffffffff,
 375         0x000008fc, 0x00000000, 0xffffffff,
 376         0x000008f8, 0x00000017, 0xffffffff,
 377         0x000008fc, 0x00000000, 0xffffffff,
 378         0x000008f8, 0x00000018, 0xffffffff,
 379         0x000008fc, 0x00000000, 0xffffffff,
 380         0x000008f8, 0x00000019, 0xffffffff,
 381         0x000008fc, 0x00000000, 0xffffffff,
 382         0x000008f8, 0x0000001a, 0xffffffff,
 383         0x000008fc, 0x00000000, 0xffffffff,
 384         0x000008f8, 0x0000001b, 0xffffffff,
 385         0x000008fc, 0x00000000, 0xffffffff,
 386         0x000008f8, 0x00000020, 0xffffffff,
 387         0x000008fc, 0xffffffff, 0xffffffff,
 388         0x000008f8, 0x00000021, 0xffffffff,
 389         0x000008fc, 0xffffffff, 0xffffffff,
 390         0x000008f8, 0x00000022, 0xffffffff,
 391         0x000008fc, 0xffffffff, 0xffffffff,
 392         0x000008f8, 0x00000023, 0xffffffff,
 393         0x000008fc, 0xffffffff, 0xffffffff,
 394         0x000008f8, 0x00000024, 0xffffffff,
 395         0x000008fc, 0xffffffff, 0xffffffff,
 396         0x000008f8, 0x00000025, 0xffffffff,
 397         0x000008fc, 0xffffffff, 0xffffffff,
 398         0x000008f8, 0x00000026, 0xffffffff,
 399         0x000008fc, 0xffffffff, 0xffffffff,
 400         0x000008f8, 0x00000027, 0xffffffff,
 401         0x000008fc, 0xffffffff, 0xffffffff,
 402         0x000008f8, 0x00000028, 0xffffffff,
 403         0x000008fc, 0xffffffff, 0xffffffff,
 404         0x000008f8, 0x00000029, 0xffffffff,
 405         0x000008fc, 0xffffffff, 0xffffffff,
 406         0x000008f8, 0x0000002a, 0xffffffff,
 407         0x000008fc, 0xffffffff, 0xffffffff,
 408         0x000008f8, 0x0000002b, 0xffffffff,
 409         0x000008fc, 0xffffffff, 0xffffffff
 410 };
 411 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 412
 413 static const u32 cayman_mgcg_default[] =
 414 {
 415         0x0000802c, 0xc0000000, 0xffffffff,
 416         0x00003fc4, 0xc0000000, 0xffffffff,
 417         0x00005448, 0x00000100, 0xffffffff,
 418         0x000055e4, 0x00000100, 0xffffffff,
 419         0x0000160c, 0x00000100, 0xffffffff,
 420         0x00008984, 0x06000100, 0xffffffff,
 421         0x0000c164, 0x00000100, 0xffffffff,
 422         0x00008a18, 0x00000100, 0xffffffff,
 423         0x0000897c, 0x06000100, 0xffffffff,
 424         0x00008b28, 0x00000100, 0xffffffff,
 425         0x00009144, 0x00800200, 0xffffffff,
 426         0x00009a60, 0x00000100, 0xffffffff,
 427         0x00009868, 0x00000100, 0xffffffff,
 428         0x00008d58, 0x00000100, 0xffffffff,
 429         0x00009510, 0x00000100, 0xffffffff,
 430         0x0000949c, 0x00000100, 0xffffffff,
 431         0x00009654, 0x00000100, 0xffffffff,
 432         0x00009030, 0x00000100, 0xffffffff,
 433         0x00009034, 0x00000100, 0xffffffff,
 434         0x00009038, 0x00000100, 0xffffffff,
 435         0x0000903c, 0x00000100, 0xffffffff,
 436         0x00009040, 0x00000100, 0xffffffff,
 437         0x0000a200, 0x00000100, 0xffffffff,
 438         0x0000a204, 0x00000100, 0xffffffff,
 439         0x0000a208, 0x00000100, 0xffffffff,
 440         0x0000a20c, 0x00000100, 0xffffffff,
 441         0x00009744, 0x00000100, 0xffffffff,
 442         0x00003f80, 0x00000100, 0xffffffff,
 443         0x0000a210, 0x00000100, 0xffffffff,
 444         0x0000a214, 0x00000100, 0xffffffff,
 445         0x000004d8, 0x00000100, 0xffffffff,
 446         0x00009664, 0x00000100, 0xffffffff,
 447         0x00009698, 0x00000100, 0xffffffff,
 448         0x000004d4, 0x00000200, 0xffffffff,
 449         0x000004d0, 0x00000000, 0xffffffff,
 450         0x000030cc, 0x00000104, 0xffffffff,
 451         0x0000d0c0, 0x00000100, 0xffffffff,
 452         0x0000d8c0, 0x00000100, 0xffffffff,
 453         0x0000802c, 0x40000000, 0xffffffff,
 454         0x00003fc4, 0x40000000, 0xffffffff,
 455         0x0000915c, 0x00010000, 0xffffffff,
 456         0x00009160, 0x00030002, 0xffffffff,
 457         0x00009164, 0x00050004, 0xffffffff,
 458         0x00009168, 0x00070006, 0xffffffff,
 459         0x00009178, 0x00070000, 0xffffffff,
 460         0x0000917c, 0x00030002, 0xffffffff,
 461         0x00009180, 0x00050004, 0xffffffff,
 462         0x0000918c, 0x00010006, 0xffffffff,
 463         0x00009190, 0x00090008, 0xffffffff,
 464         0x00009194, 0x00070000, 0xffffffff,
 465         0x00009198, 0x00030002, 0xffffffff,
 466         0x0000919c, 0x00050004, 0xffffffff,
 467         0x000091a8, 0x00010006, 0xffffffff,
 468         0x000091ac, 0x00090008, 0xffffffff,
 469         0x000091b0, 0x00070000, 0xffffffff,
 470         0x000091b4, 0x00030002, 0xffffffff,
 471         0x000091b8, 0x00050004, 0xffffffff,
 472         0x000091c4, 0x00010006, 0xffffffff,
 473         0x000091c8, 0x00090008, 0xffffffff,
 474         0x000091cc, 0x00070000, 0xffffffff,
 475         0x000091d0, 0x00030002, 0xffffffff,
 476         0x000091d4, 0x00050004, 0xffffffff,
 477         0x000091e0, 0x00010006, 0xffffffff,
 478         0x000091e4, 0x00090008, 0xffffffff,
 479         0x000091e8, 0x00000000, 0xffffffff,
 480         0x000091ec, 0x00070000, 0xffffffff,
 481         0x000091f0, 0x00030002, 0xffffffff,
 482         0x000091f4, 0x00050004, 0xffffffff,
 483         0x00009200, 0x00010006, 0xffffffff,
 484         0x00009204, 0x00090008, 0xffffffff,
 485         0x00009208, 0x00070000, 0xffffffff,
 486         0x0000920c, 0x00030002, 0xffffffff,
 487         0x00009210, 0x00050004, 0xffffffff,
 488         0x0000921c, 0x00010006, 0xffffffff,
 489         0x00009220, 0x00090008, 0xffffffff,
 490         0x00009224, 0x00070000, 0xffffffff,
 491         0x00009228, 0x00030002, 0xffffffff,
 492         0x0000922c, 0x00050004, 0xffffffff,
 493         0x00009238, 0x00010006, 0xffffffff,
 494         0x0000923c, 0x00090008, 0xffffffff,
 495         0x00009240, 0x00070000, 0xffffffff,
 496         0x00009244, 0x00030002, 0xffffffff,
 497         0x00009248, 0x00050004, 0xffffffff,
 498         0x00009254, 0x00010006, 0xffffffff,
 499         0x00009258, 0x00090008, 0xffffffff,
 500         0x0000925c, 0x00070000, 0xffffffff,
 501         0x00009260, 0x00030002, 0xffffffff,
 502         0x00009264, 0x00050004, 0xffffffff,
 503         0x00009270, 0x00010006, 0xffffffff,
 504         0x00009274, 0x00090008, 0xffffffff,
 505         0x00009278, 0x00070000, 0xffffffff,
 506         0x0000927c, 0x00030002, 0xffffffff,
 507         0x00009280, 0x00050004, 0xffffffff,
 508         0x0000928c, 0x00010006, 0xffffffff,
 509         0x00009290, 0x00090008, 0xffffffff,
 510         0x000092a8, 0x00070000, 0xffffffff,
 511         0x000092ac, 0x00030002, 0xffffffff,
 512         0x000092b0, 0x00050004, 0xffffffff,
 513         0x000092bc, 0x00010006, 0xffffffff,
 514         0x000092c0, 0x00090008, 0xffffffff,
 515         0x000092c4, 0x00070000, 0xffffffff,
 516         0x000092c8, 0x00030002, 0xffffffff,
 517         0x000092cc, 0x00050004, 0xffffffff,
 518         0x000092d8, 0x00010006, 0xffffffff,
 519         0x000092dc, 0x00090008, 0xffffffff,
 520         0x00009294, 0x00000000, 0xffffffff,
 521         0x0000802c, 0x40010000, 0xffffffff,
 522         0x00003fc4, 0x40010000, 0xffffffff,
 523         0x0000915c, 0x00010000, 0xffffffff,
 524         0x00009160, 0x00030002, 0xffffffff,
 525         0x00009164, 0x00050004, 0xffffffff,
 526         0x00009168, 0x00070006, 0xffffffff,
 527         0x00009178, 0x00070000, 0xffffffff,
 528         0x0000917c, 0x00030002, 0xffffffff,
 529         0x00009180, 0x00050004, 0xffffffff,
 530         0x0000918c, 0x00010006, 0xffffffff,
 531         0x00009190, 0x00090008, 0xffffffff,
 532         0x00009194, 0x00070000, 0xffffffff,
 533         0x00009198, 0x00030002, 0xffffffff,
 534         0x0000919c, 0x00050004, 0xffffffff,
 535         0x000091a8, 0x00010006, 0xffffffff,
 536         0x000091ac, 0x00090008, 0xffffffff,
 537         0x000091b0, 0x00070000, 0xffffffff,
 538         0x000091b4, 0x00030002, 0xffffffff,
 539         0x000091b8, 0x00050004, 0xffffffff,
 540         0x000091c4, 0x00010006, 0xffffffff,
 541         0x000091c8, 0x00090008, 0xffffffff,
 542         0x000091cc, 0x00070000, 0xffffffff,
 543         0x000091d0, 0x00030002, 0xffffffff,
 544         0x000091d4, 0x00050004, 0xffffffff,
 545         0x000091e0, 0x00010006, 0xffffffff,
 546         0x000091e4, 0x00090008, 0xffffffff,
 547         0x000091e8, 0x00000000, 0xffffffff,
 548         0x000091ec, 0x00070000, 0xffffffff,
 549         0x000091f0, 0x00030002, 0xffffffff,
 550         0x000091f4, 0x00050004, 0xffffffff,
 551         0x00009200, 0x00010006, 0xffffffff,
 552         0x00009204, 0x00090008, 0xffffffff,
 553         0x00009208, 0x00070000, 0xffffffff,
 554         0x0000920c, 0x00030002, 0xffffffff,
 555         0x00009210, 0x00050004, 0xffffffff,
 556         0x0000921c, 0x00010006, 0xffffffff,
 557         0x00009220, 0x00090008, 0xffffffff,
 558         0x00009224, 0x00070000, 0xffffffff,
 559         0x00009228, 0x00030002, 0xffffffff,
 560         0x0000922c, 0x00050004, 0xffffffff,
 561         0x00009238, 0x00010006, 0xffffffff,
 562         0x0000923c, 0x00090008, 0xffffffff,
 563         0x00009240, 0x00070000, 0xffffffff,
 564         0x00009244, 0x00030002, 0xffffffff,
 565         0x00009248, 0x00050004, 0xffffffff,
 566         0x00009254, 0x00010006, 0xffffffff,
 567         0x00009258, 0x00090008, 0xffffffff,
 568         0x0000925c, 0x00070000, 0xffffffff,
 569         0x00009260, 0x00030002, 0xffffffff,
 570         0x00009264, 0x00050004, 0xffffffff,
 571         0x00009270, 0x00010006, 0xffffffff,
 572         0x00009274, 0x00090008, 0xffffffff,
 573         0x00009278, 0x00070000, 0xffffffff,
 574         0x0000927c, 0x00030002, 0xffffffff,
 575         0x00009280, 0x00050004, 0xffffffff,
 576         0x0000928c, 0x00010006, 0xffffffff,
 577         0x00009290, 0x00090008, 0xffffffff,
 578         0x000092a8, 0x00070000, 0xffffffff,
 579         0x000092ac, 0x00030002, 0xffffffff,
 580         0x000092b0, 0x00050004, 0xffffffff,
 581         0x000092bc, 0x00010006, 0xffffffff,
 582         0x000092c0, 0x00090008, 0xffffffff,
 583         0x000092c4, 0x00070000, 0xffffffff,
 584         0x000092c8, 0x00030002, 0xffffffff,
 585         0x000092cc, 0x00050004, 0xffffffff,
 586         0x000092d8, 0x00010006, 0xffffffff,
 587         0x000092dc, 0x00090008, 0xffffffff,
 588         0x00009294, 0x00000000, 0xffffffff,
 589         0x0000802c, 0xc0000000, 0xffffffff,
 590         0x00003fc4, 0xc0000000, 0xffffffff,
 591         0x000008f8, 0x00000010, 0xffffffff,
 592         0x000008fc, 0x00000000, 0xffffffff,
 593         0x000008f8, 0x00000011, 0xffffffff,
 594         0x000008fc, 0x00000000, 0xffffffff,
 595         0x000008f8, 0x00000012, 0xffffffff,
 596         0x000008fc, 0x00000000, 0xffffffff,
 597         0x000008f8, 0x00000013, 0xffffffff,
 598         0x000008fc, 0x00000000, 0xffffffff,
 599         0x000008f8, 0x00000014, 0xffffffff,
 600         0x000008fc, 0x00000000, 0xffffffff,
 601         0x000008f8, 0x00000015, 0xffffffff,
 602         0x000008fc, 0x00000000, 0xffffffff,
 603         0x000008f8, 0x00000016, 0xffffffff,
 604         0x000008fc, 0x00000000, 0xffffffff,
 605         0x000008f8, 0x00000017, 0xffffffff,
 606         0x000008fc, 0x00000000, 0xffffffff,
 607         0x000008f8, 0x00000018, 0xffffffff,
 608         0x000008fc, 0x00000000, 0xffffffff,
 609         0x000008f8, 0x00000019, 0xffffffff,
 610         0x000008fc, 0x00000000, 0xffffffff,
 611         0x000008f8, 0x0000001a, 0xffffffff,
 612         0x000008fc, 0x00000000, 0xffffffff,
 613         0x000008f8, 0x0000001b, 0xffffffff,
 614         0x000008fc, 0x00000000, 0xffffffff
 615 };
 616 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 617
 618 static const u32 cayman_mgcg_disable[] =
 619 {
 620         0x0000802c, 0xc0000000, 0xffffffff,
 621         0x000008f8, 0x00000000, 0xffffffff,
 622         0x000008fc, 0xffffffff, 0xffffffff,
 623         0x000008f8, 0x00000001, 0xffffffff,
 624         0x000008fc, 0xffffffff, 0xffffffff,
 625         0x000008f8, 0x00000002, 0xffffffff,
 626         0x000008fc, 0xffffffff, 0xffffffff,
 627         0x000008f8, 0x00000003, 0xffffffff,
 628         0x000008fc, 0xffffffff, 0xffffffff,
 629         0x00009150, 0x00600000, 0xffffffff
 630 };
 631 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 632
 633 static const u32 cayman_mgcg_enable[] =
 634 {
 635         0x0000802c, 0xc0000000, 0xffffffff,
 636         0x000008f8, 0x00000000, 0xffffffff,
 637         0x000008fc, 0x00000000, 0xffffffff,
 638         0x000008f8, 0x00000001, 0xffffffff,
 639         0x000008fc, 0x00000000, 0xffffffff,
 640         0x000008f8, 0x00000002, 0xffffffff,
 641         0x000008fc, 0x00600000, 0xffffffff,
 642         0x000008f8, 0x00000003, 0xffffffff,
 643         0x000008fc, 0x00000000, 0xffffffff,
 644         0x00009150, 0x96944200, 0xffffffff
 645 };
 646
 647 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 648
 649 #define NISLANDS_SYSLS_SEQUENCE  100
 650
 651 static const u32 cayman_sysls_default[] =
 652 {
 653         /* Register,   Value,     Mask bits */
 654         0x000055e8, 0x00000000, 0xffffffff,
 655         0x0000d0bc, 0x00000000, 0xffffffff,
 656         0x0000d8bc, 0x00000000, 0xffffffff,
 657         0x000015c0, 0x000c1401, 0xffffffff,
 658         0x0000264c, 0x000c0400, 0xffffffff,
 659         0x00002648, 0x000c0400, 0xffffffff,
 660         0x00002650, 0x000c0400, 0xffffffff,
 661         0x000020b8, 0x000c0400, 0xffffffff,
 662         0x000020bc, 0x000c0400, 0xffffffff,
 663         0x000020c0, 0x000c0c80, 0xffffffff,
 664         0x0000f4a0, 0x000000c0, 0xffffffff,
 665         0x0000f4a4, 0x00680fff, 0xffffffff,
 666         0x00002f50, 0x00000404, 0xffffffff,
 667         0x000004c8, 0x00000001, 0xffffffff,
 668         0x000064ec, 0x00000000, 0xffffffff,
 669         0x00000c7c, 0x00000000, 0xffffffff,
 670         0x00008dfc, 0x00000000, 0xffffffff
 671 };
 672 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 673
 674 static const u32 cayman_sysls_disable[] =
 675 {
 676         /* Register,   Value,     Mask bits */
 677         0x0000d0c0, 0x00000000, 0xffffffff,
 678         0x0000d8c0, 0x00000000, 0xffffffff,
 679         0x000055e8, 0x00000000, 0xffffffff,
 680         0x0000d0bc, 0x00000000, 0xffffffff,
 681         0x0000d8bc, 0x00000000, 0xffffffff,
 682         0x000015c0, 0x00041401, 0xffffffff,
 683         0x0000264c, 0x00040400, 0xffffffff,
 684         0x00002648, 0x00040400, 0xffffffff,
 685         0x00002650, 0x00040400, 0xffffffff,
 686         0x000020b8, 0x00040400, 0xffffffff,
 687         0x000020bc, 0x00040400, 0xffffffff,
 688         0x000020c0, 0x00040c80, 0xffffffff,
 689         0x0000f4a0, 0x000000c0, 0xffffffff,
 690         0x0000f4a4, 0x00680000, 0xffffffff,
 691         0x00002f50, 0x00000404, 0xffffffff,
 692         0x000004c8, 0x00000001, 0xffffffff,
 693         0x000064ec, 0x00007ffd, 0xffffffff,
 694         0x00000c7c, 0x0000ff00, 0xffffffff,
 695         0x00008dfc, 0x0000007f, 0xffffffff
 696 };
 697 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 698
 699 static const u32 cayman_sysls_enable[] =
 700 {
 701         /* Register,   Value,     Mask bits */
 702         0x000055e8, 0x00000001, 0xffffffff,
 703         0x0000d0bc, 0x00000100, 0xffffffff,
 704         0x0000d8bc, 0x00000100, 0xffffffff,
 705         0x000015c0, 0x000c1401, 0xffffffff,
 706         0x0000264c, 0x000c0400, 0xffffffff,
 707         0x00002648, 0x000c0400, 0xffffffff,
 708         0x00002650, 0x000c0400, 0xffffffff,
 709         0x000020b8, 0x000c0400, 0xffffffff,
 710         0x000020bc, 0x000c0400, 0xffffffff,
 711         0x000020c0, 0x000c0c80, 0xffffffff,
 712         0x0000f4a0, 0x000000c0, 0xffffffff,
 713         0x0000f4a4, 0x00680fff, 0xffffffff,
 714         0x00002f50, 0x00000903, 0xffffffff,
 715         0x000004c8, 0x00000000, 0xffffffff,
 716         0x000064ec, 0x00000000, 0xffffffff,
 717         0x00000c7c, 0x00000000, 0xffffffff,
 718         0x00008dfc, 0x00000000, 0xffffffff
 719 };
 720 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 721
 722 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 723 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 724
 725 extern int ni_mc_load_microcode(struct radeon_device *rdev);
 726
 727 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 728 {
 729         struct ni_power_info *pi = rdev->pm.dpm.priv;
 730
 731         return pi;
 732 }
 733
 734 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 735 {
 736         struct ni_ps *ps = rps->ps_priv;
 737
 738         return ps;
 739 }
 740
 741 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 742                                                      u16 v, s32 t,
 743                                                      u32 ileakage,
 744                                                      u32 *leakage)
 745 {
 746         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 747
 748         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 749         vddc = div64_s64(drm_int2fixp(v), 1000);
 750         temperature = div64_s64(drm_int2fixp(t), 1000);
 751
 752         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 753                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 754         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 755                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 756
 757         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 758
 759         *leakage = drm_fixp2int(leakage_w * 1000);
 760 }
 761
 762 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 763                                              const struct ni_leakage_coeffients *coeff,
 764                                              u16 v,
 765                                              s32 t,
 766                                              u32 i_leakage,
 767                                              u32 *leakage)
 768 {
 769         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 770 }
 771
 772 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
 773 {
 774         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 775         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 776         /* we never hit the non-gddr5 limit so disable it */
 777         u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
 778
 779         if (vblank_time < switch_limit)
 780                 return true;
 781         else
 782                 return false;
 783
 784 }
 785
 786 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 787                                         struct radeon_ps *rps)
 788 {
 789         struct ni_ps *ps = ni_get_ps(rps);
 790         struct radeon_clock_and_voltage_limits *max_limits;
 791         bool disable_mclk_switching;
 792         u32 mclk;
 793         u16 vddci;
 794         int i;
 795
 796         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 797             ni_dpm_vblank_too_short(rdev))
 798                 disable_mclk_switching = true;
 799         else
 800                 disable_mclk_switching = false;
 801
 802         if (rdev->pm.dpm.ac_power)
 803                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 804         else
 805                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 806
 807         if (rdev->pm.dpm.ac_power == false) {
 808                 for (i = 0; i < ps->performance_level_count; i++) {
 809                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 810                                 ps->performance_levels[i].mclk = max_limits->mclk;
 811                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 812                                 ps->performance_levels[i].sclk = max_limits->sclk;
 813                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 814                                 ps->performance_levels[i].vddc = max_limits->vddc;
 815                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 816                                 ps->performance_levels[i].vddci = max_limits->vddci;
 817                 }
 818         }
 819
 820         /* XXX validate the min clocks required for display */
 821
 822         /* adjust low state */
 823         if (disable_mclk_switching) {
 824                 ps->performance_levels[0].mclk =
 825                         ps->performance_levels[ps->performance_level_count - 1].mclk;
 826                 ps->performance_levels[0].vddci =
 827                         ps->performance_levels[ps->performance_level_count - 1].vddci;
 828         }
 829
 830         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 831                                   &ps->performance_levels[0].sclk,
 832                                   &ps->performance_levels[0].mclk);
 833
 834         for (i = 1; i < ps->performance_level_count; i++) {
 835                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 836                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 837                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 838                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 839         }
 840
 841         /* adjust remaining states */
 842         if (disable_mclk_switching) {
 843                 mclk = ps->performance_levels[0].mclk;
 844                 vddci = ps->performance_levels[0].vddci;
 845                 for (i = 1; i < ps->performance_level_count; i++) {
 846                         if (mclk < ps->performance_levels[i].mclk)
 847                                 mclk = ps->performance_levels[i].mclk;
 848                         if (vddci < ps->performance_levels[i].vddci)
 849                                 vddci = ps->performance_levels[i].vddci;
 850                 }
 851                 for (i = 0; i < ps->performance_level_count; i++) {
 852                         ps->performance_levels[i].mclk = mclk;
 853                         ps->performance_levels[i].vddci = vddci;
 854                 }
 855         } else {
 856                 for (i = 1; i < ps->performance_level_count; i++) {
 857                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 858                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 859                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 860                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 861                 }
 862         }
 863
 864         for (i = 1; i < ps->performance_level_count; i++)
 865                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 866                                           &ps->performance_levels[i].sclk,
 867                                           &ps->performance_levels[i].mclk);
 868
 869         for (i = 0; i < ps->performance_level_count; i++)
 870                 btc_adjust_clock_combinations(rdev, max_limits,
 871                                               &ps->performance_levels[i]);
 872
 873         for (i = 0; i < ps->performance_level_count; i++) {
 874                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 875                                                    ps->performance_levels[i].sclk,
 876                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 877                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 878                                                    ps->performance_levels[i].mclk,
 879                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 880                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 881                                                    ps->performance_levels[i].mclk,
 882                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 883                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 884                                                    rdev->clock.current_dispclk,
 885                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 886         }
 887
 888         for (i = 0; i < ps->performance_level_count; i++) {
 889                 btc_apply_voltage_delta_rules(rdev,
 890                                               max_limits->vddc, max_limits->vddci,
 891                                               &ps->performance_levels[i].vddc,
 892                                               &ps->performance_levels[i].vddci);
 893         }
 894
 895         ps->dc_compatible = true;
 896         for (i = 0; i < ps->performance_level_count; i++) {
 897                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 898                         ps->dc_compatible = false;
 899
 900                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 901                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 902         }
 903 }
 904
 905 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 906 {
 907         u32 count;
 908         const u32 *ps = NULL;
 909
 910         ps = (const u32 *)&cayman_cgcg_cgls_default;
 911         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 912
 913         btc_program_mgcg_hw_sequence(rdev, ps, count);
 914 }
 915
 916 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 917                                       bool enable)
 918 {
 919         u32 count;
 920         const u32 *ps = NULL;
 921
 922         if (enable) {
 923                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 924                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 925         } else {
 926                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 927                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 928         }
 929
 930         btc_program_mgcg_hw_sequence(rdev, ps, count);
 931 }
 932
 933 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 934 {
 935         u32 count;
 936         const u32 *ps = NULL;
 937
 938         ps = (const u32 *)&cayman_mgcg_default;
 939         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 940
 941         btc_program_mgcg_hw_sequence(rdev, ps, count);
 942 }
 943
 944 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 945                                      bool enable)
 946 {
 947         u32 count;
 948         const u32 *ps = NULL;
 949
 950         if (enable) {
 951                 ps = (const u32 *)&cayman_mgcg_enable;
 952                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 953         } else {
 954                 ps = (const u32 *)&cayman_mgcg_disable;
 955                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 956         }
 957
 958         btc_program_mgcg_hw_sequence(rdev, ps, count);
 959 }
 960
 961 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 962 {
 963         u32 count;
 964         const u32 *ps = NULL;
 965
 966         ps = (const u32 *)&cayman_sysls_default;
 967         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 968
 969         btc_program_mgcg_hw_sequence(rdev, ps, count);
 970 }
 971
 972 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 973                                      bool enable)
 974 {
 975         u32 count;
 976         const u32 *ps = NULL;
 977
 978         if (enable) {
 979                 ps = (const u32 *)&cayman_sysls_enable;
 980                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
 981         } else {
 982                 ps = (const u32 *)&cayman_sysls_disable;
 983                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
 984         }
 985
 986         btc_program_mgcg_hw_sequence(rdev, ps, count);
 987
 988 }
 989
 990 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
 991                                                              struct radeon_clock_voltage_dependency_table *table)
 992 {
 993         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 994         u32 i;
 995
 996         if (table) {
 997                 for (i = 0; i < table->count; i++) {
 998                         if (0xff01 == table->entries[i].v) {
 999                                 if (pi->max_vddc == 0)
1000                                         return -EINVAL;
1001                                 table->entries[i].v = pi->max_vddc;
1002                         }
1003                 }
1004         }
1005         return 0;
1006 }
1007
1008 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1009 {
1010         int ret = 0;
1011
1012         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1013                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1014
1015         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1016                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1017         return ret;
1018 }
1019
1020 static void ni_stop_dpm(struct radeon_device *rdev)
1021 {
1022         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1023 }
1024
1025 #if 0
1026 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1027                                         bool ac_power)
1028 {
1029         if (ac_power)
1030                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1031                         0 : -EINVAL;
1032
1033         return 0;
1034 }
1035 #endif
1036
1037 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1038                                                       PPSMC_Msg msg, u32 parameter)
1039 {
1040         WREG32(SMC_SCRATCH0, parameter);
1041         return rv770_send_msg_to_smc(rdev, msg);
1042 }
1043
1044 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1045 {
1046         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1047                 return -EINVAL;
1048
1049         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1050                 0 : -EINVAL;
1051 }
1052
1053 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1054                                    enum radeon_dpm_forced_level level)
1055 {
1056         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1057                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1058                         return -EINVAL;
1059
1060                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1061                         return -EINVAL;
1062         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1063                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1064                         return -EINVAL;
1065
1066                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1067                         return -EINVAL;
1068         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1069                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1070                         return -EINVAL;
1071
1072                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1073                         return -EINVAL;
1074         }
1075
1076         rdev->pm.dpm.forced_level = level;
1077
1078         return 0;
1079 }
1080
1081 static void ni_stop_smc(struct radeon_device *rdev)
1082 {
1083         u32 tmp;
1084         int i;
1085
1086         for (i = 0; i < rdev->usec_timeout; i++) {
1087                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1088                 if (tmp != 1)
1089                         break;
1090                 udelay(1);
1091         }
1092
1093         udelay(100);
1094
1095         r7xx_stop_smc(rdev);
1096 }
1097
1098 static int ni_process_firmware_header(struct radeon_device *rdev)
1099 {
1100         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1101         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1102         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1103         u32 tmp;
1104         int ret;
1105
1106         ret = rv770_read_smc_sram_dword(rdev,
1107                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1108                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1109                                         &tmp, pi->sram_end);
1110
1111         if (ret)
1112                 return ret;
1113
1114         pi->state_table_start = (u16)tmp;
1115
1116         ret = rv770_read_smc_sram_dword(rdev,
1117                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1118                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1119                                         &tmp, pi->sram_end);
1120
1121         if (ret)
1122                 return ret;
1123
1124         pi->soft_regs_start = (u16)tmp;
1125
1126         ret = rv770_read_smc_sram_dword(rdev,
1127                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1128                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1129                                         &tmp, pi->sram_end);
1130
1131         if (ret)
1132                 return ret;
1133
1134         eg_pi->mc_reg_table_start = (u16)tmp;
1135
1136         ret = rv770_read_smc_sram_dword(rdev,
1137                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1138                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1139                                         &tmp, pi->sram_end);
1140
1141         if (ret)
1142                 return ret;
1143
1144         ni_pi->fan_table_start = (u16)tmp;
1145
1146         ret = rv770_read_smc_sram_dword(rdev,
1147                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1148                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1149                                         &tmp, pi->sram_end);
1150
1151         if (ret)
1152                 return ret;
1153
1154         ni_pi->arb_table_start = (u16)tmp;
1155
1156         ret = rv770_read_smc_sram_dword(rdev,
1157                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1158                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1159                                         &tmp, pi->sram_end);
1160
1161         if (ret)
1162                 return ret;
1163
1164         ni_pi->cac_table_start = (u16)tmp;
1165
1166         ret = rv770_read_smc_sram_dword(rdev,
1167                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1168                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1169                                         &tmp, pi->sram_end);
1170
1171         if (ret)
1172                 return ret;
1173
1174         ni_pi->spll_table_start = (u16)tmp;
1175
1176
1177         return ret;
1178 }
1179
1180 static void ni_read_clock_registers(struct radeon_device *rdev)
1181 {
1182         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1183
1184         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1185         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1186         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1187         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1188         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1189         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1190         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1191         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1192         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1193         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1194         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1195         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1196         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1197         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1198 }
1199
1200 #if 0
1201 static int ni_enter_ulp_state(struct radeon_device *rdev)
1202 {
1203         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1204
1205         if (pi->gfx_clock_gating) {
1206                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1207                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1208                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1209                 RREG32(GB_ADDR_CONFIG);
1210         }
1211
1212         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1213                  ~HOST_SMC_MSG_MASK);
1214
1215         udelay(25000);
1216
1217         return 0;
1218 }
1219 #endif
1220
1221 static void ni_program_response_times(struct radeon_device *rdev)
1222 {
1223         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1224         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1225         u32 reference_clock;
1226
1227         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1228
1229         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1230         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1231
1232         if (voltage_response_time == 0)
1233                 voltage_response_time = 1000;
1234
1235         if (backbias_response_time == 0)
1236                 backbias_response_time = 1000;
1237
1238         acpi_delay_time = 15000;
1239         vbi_time_out = 100000;
1240
1241         reference_clock = radeon_get_xclk(rdev);
1242
1243         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1244         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1245         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1246         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1247
1248         mclk_switch_limit = (460 * reference_clock) / 100;
1249
1250         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1251         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1252         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1253         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1254         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1255         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1256 }
1257
1258 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1259                                           struct atom_voltage_table *voltage_table,
1260                                           NISLANDS_SMC_STATETABLE *table)
1261 {
1262         unsigned int i;
1263
1264         for (i = 0; i < voltage_table->count; i++) {
1265                 table->highSMIO[i] = 0;
1266                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1267         }
1268 }
1269
1270 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1271                                            NISLANDS_SMC_STATETABLE *table)
1272 {
1273         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1274         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1275         unsigned char i;
1276
1277         if (eg_pi->vddc_voltage_table.count) {
1278                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1279                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1280                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1281                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1282
1283                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1284                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1285                                 table->maxVDDCIndexInPPTable = i;
1286                                 break;
1287                         }
1288                 }
1289         }
1290
1291         if (eg_pi->vddci_voltage_table.count) {
1292                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1293
1294                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1295                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1296                         cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
1297         }
1298 }
1299
1300 static int ni_populate_voltage_value(struct radeon_device *rdev,
1301                                      struct atom_voltage_table *table,
1302                                      u16 value,
1303                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1304 {
1305         unsigned int i;
1306
1307         for (i = 0; i < table->count; i++) {
1308                 if (value <= table->entries[i].value) {
1309                         voltage->index = (u8)i;
1310                         voltage->value = cpu_to_be16(table->entries[i].value);
1311                         break;
1312                 }
1313         }
1314
1315         if (i >= table->count)
1316                 return -EINVAL;
1317
1318         return 0;
1319 }
1320
1321 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1322                                    u32 mclk,
1323                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1324 {
1325         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1326         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1327
1328         if (!pi->mvdd_control) {
1329                 voltage->index = eg_pi->mvdd_high_index;
1330                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1331                 return;
1332         }
1333
1334         if (mclk <= pi->mvdd_split_frequency) {
1335                 voltage->index = eg_pi->mvdd_low_index;
1336                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1337         } else {
1338                 voltage->index = eg_pi->mvdd_high_index;
1339                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1340         }
1341 }
1342
1343 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1344                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1345                                     u16 *std_voltage)
1346 {
1347         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1348             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1349                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1350         else
1351                 *std_voltage = be16_to_cpu(voltage->value);
1352
1353         return 0;
1354 }
1355
1356 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1357                                           u16 value, u8 index,
1358                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1359 {
1360         voltage->index = index;
1361         voltage->value = cpu_to_be16(value);
1362 }
1363
1364 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1365 {
1366         u32 xclk_period;
1367         u32 xclk = radeon_get_xclk(rdev);
1368         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1369
1370         xclk_period = (1000000000UL / xclk);
1371         xclk_period /= 10000UL;
1372
1373         return tmp * xclk_period;
1374 }
1375
1376 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1377 {
1378         return (power_in_watts * scaling_factor) << 2;
1379 }
1380
1381 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1382                                           struct radeon_ps *radeon_state,
1383                                           u32 near_tdp_limit)
1384 {
1385         struct ni_ps *state = ni_get_ps(radeon_state);
1386         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1387         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1388         u32 power_boost_limit = 0;
1389         int ret;
1390
1391         if (ni_pi->enable_power_containment &&
1392             ni_pi->use_power_boost_limit) {
1393                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1394                 u16 std_vddc_med;
1395                 u16 std_vddc_high;
1396                 u64 tmp, n, d;
1397
1398                 if (state->performance_level_count < 3)
1399                         return 0;
1400
1401                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1402                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1403                                                 &vddc);
1404                 if (ret)
1405                         return 0;
1406
1407                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1408                 if (ret)
1409                         return 0;
1410
1411                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1412                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1413                                                 &vddc);
1414                 if (ret)
1415                         return 0;
1416
1417                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1418                 if (ret)
1419                         return 0;
1420
1421                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1422                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1423                 tmp = div64_u64(n, d);
1424
1425                 if (tmp >> 32)
1426                         return 0;
1427                 power_boost_limit = (u32)tmp;
1428         }
1429
1430         return power_boost_limit;
1431 }
1432
1433 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1434                                             bool adjust_polarity,
1435                                             u32 tdp_adjustment,
1436                                             u32 *tdp_limit,
1437                                             u32 *near_tdp_limit)
1438 {
1439         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1440                 return -EINVAL;
1441
1442         if (adjust_polarity) {
1443                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1444                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1445         } else {
1446                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1447                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1448         }
1449
1450         return 0;
1451 }
1452
1453 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1454                                       struct radeon_ps *radeon_state)
1455 {
1456         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1457         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1458
1459         if (ni_pi->enable_power_containment) {
1460                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1461                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1462                 u32 tdp_limit;
1463                 u32 near_tdp_limit;
1464                 u32 power_boost_limit;
1465                 int ret;
1466
1467                 if (scaling_factor == 0)
1468                         return -EINVAL;
1469
1470                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1471
1472                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1473                                                        false, /* ??? */
1474                                                        rdev->pm.dpm.tdp_adjustment,
1475                                                        &tdp_limit,
1476                                                        &near_tdp_limit);
1477                 if (ret)
1478                         return ret;
1479
1480                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1481                                                                    near_tdp_limit);
1482
1483                 smc_table->dpm2Params.TDPLimit =
1484                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1485                 smc_table->dpm2Params.NearTDPLimit =
1486                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1487                 smc_table->dpm2Params.SafePowerLimit =
1488                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1489                                                            scaling_factor));
1490                 smc_table->dpm2Params.PowerBoostLimit =
1491                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1492
1493                 ret = rv770_copy_bytes_to_smc(rdev,
1494                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1495                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1496                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1497                                               sizeof(u32) * 4, pi->sram_end);
1498                 if (ret)
1499                         return ret;
1500         }
1501
1502         return 0;
1503 }
1504
1505 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1506                                 u32 arb_freq_src, u32 arb_freq_dest)
1507 {
1508         u32 mc_arb_dram_timing;
1509         u32 mc_arb_dram_timing2;
1510         u32 burst_time;
1511         u32 mc_cg_config;
1512
1513         switch (arb_freq_src) {
1514         case MC_CG_ARB_FREQ_F0:
1515                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1516                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1517                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1518                 break;
1519         case MC_CG_ARB_FREQ_F1:
1520                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1521                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1522                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1523                 break;
1524         case MC_CG_ARB_FREQ_F2:
1525                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1526                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1527                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1528                 break;
1529         case MC_CG_ARB_FREQ_F3:
1530                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1531                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1532                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1533                 break;
1534         default:
1535                 return -EINVAL;
1536         }
1537
1538         switch (arb_freq_dest) {
1539         case MC_CG_ARB_FREQ_F0:
1540                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1541                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1542                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1543                 break;
1544         case MC_CG_ARB_FREQ_F1:
1545                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1546                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1547                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1548                 break;
1549         case MC_CG_ARB_FREQ_F2:
1550                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1551                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1552                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1553                 break;
1554         case MC_CG_ARB_FREQ_F3:
1555                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1556                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1557                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1558                 break;
1559         default:
1560                 return -EINVAL;
1561         }
1562
1563         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1564         WREG32(MC_CG_CONFIG, mc_cg_config);
1565         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1566
1567         return 0;
1568 }
1569
1570 static int ni_init_arb_table_index(struct radeon_device *rdev)
1571 {
1572         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1573         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1574         u32 tmp;
1575         int ret;
1576
1577         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1578                                         &tmp, pi->sram_end);
1579         if (ret)
1580                 return ret;
1581
1582         tmp &= 0x00FFFFFF;
1583         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1584
1585         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1586                                           tmp, pi->sram_end);
1587 }
1588
1589 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1590 {
1591         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1592 }
1593
1594 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1595 {
1596         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1597         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1598         u32 tmp;
1599         int ret;
1600
1601         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1602                                         &tmp, pi->sram_end);
1603         if (ret)
1604                 return ret;
1605
1606         tmp = (tmp >> 24) & 0xff;
1607
1608         if (tmp == MC_CG_ARB_FREQ_F0)
1609                 return 0;
1610
1611         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1612 }
1613
1614 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1615                                                 struct rv7xx_pl *pl,
1616                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1617 {
1618         u32 dram_timing;
1619         u32 dram_timing2;
1620
1621         arb_regs->mc_arb_rfsh_rate =
1622                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1623
1624
1625         radeon_atom_set_engine_dram_timings(rdev, pl->sclk, pl->mclk);
1626
1627         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1628         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1629
1630         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1631         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1632
1633         return 0;
1634 }
1635
1636 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1637                                                   struct radeon_ps *radeon_state,
1638                                                   unsigned int first_arb_set)
1639 {
1640         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1641         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1642         struct ni_ps *state = ni_get_ps(radeon_state);
1643         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1644         int i, ret = 0;
1645
1646         for (i = 0; i < state->performance_level_count; i++) {
1647                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1648                 if (ret)
1649                         break;
1650
1651                 ret = rv770_copy_bytes_to_smc(rdev,
1652                                               (u16)(ni_pi->arb_table_start +
1653                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1654                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1655                                               (u8 *)&arb_regs,
1656                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1657                                               pi->sram_end);
1658                 if (ret)
1659                         break;
1660         }
1661         return ret;
1662 }
1663
1664 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1665                                                struct radeon_ps *radeon_new_state)
1666 {
1667         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1668                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1669 }
1670
1671 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1672                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1673 {
1674         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1675
1676         voltage->index = eg_pi->mvdd_high_index;
1677         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1678 }
1679
1680 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1681                                          struct radeon_ps *radeon_initial_state,
1682                                          NISLANDS_SMC_STATETABLE *table)
1683 {
1684         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1685         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1686         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1687         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1688         u32 reg;
1689         int ret;
1690
1691         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1692                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1693         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1694                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1695         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1696                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1697         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1698                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1699         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1700                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1701         table->initialState.levels[0].mclk.vDLL_CNTL =
1702                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1703         table->initialState.levels[0].mclk.vMPLL_SS =
1704                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1705         table->initialState.levels[0].mclk.vMPLL_SS2 =
1706                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1707         table->initialState.levels[0].mclk.mclk_value =
1708                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1709
1710         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1711                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1712         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1713                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1714         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1715                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1716         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1717                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1718         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1719                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1720         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1721                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1722         table->initialState.levels[0].sclk.sclk_value =
1723                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1724         table->initialState.levels[0].arbRefreshState =
1725                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1726
1727         table->initialState.levels[0].ACIndex = 0;
1728
1729         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1730                                         initial_state->performance_levels[0].vddc,
1731                                         &table->initialState.levels[0].vddc);
1732         if (!ret) {
1733                 u16 std_vddc;
1734
1735                 ret = ni_get_std_voltage_value(rdev,
1736                                                &table->initialState.levels[0].vddc,
1737                                                &std_vddc);
1738                 if (!ret)
1739                         ni_populate_std_voltage_value(rdev, std_vddc,
1740                                                       table->initialState.levels[0].vddc.index,
1741                                                       &table->initialState.levels[0].std_vddc);
1742         }
1743
1744         if (eg_pi->vddci_control)
1745                 ni_populate_voltage_value(rdev,
1746                                           &eg_pi->vddci_voltage_table,
1747                                           initial_state->performance_levels[0].vddci,
1748                                           &table->initialState.levels[0].vddci);
1749
1750         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1751
1752         reg = CG_R(0xffff) | CG_L(0);
1753         table->initialState.levels[0].aT = cpu_to_be32(reg);
1754
1755         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1756
1757         if (pi->boot_in_gen2)
1758                 table->initialState.levels[0].gen2PCIE = 1;
1759         else
1760                 table->initialState.levels[0].gen2PCIE = 0;
1761
1762         if (pi->mem_gddr5) {
1763                 table->initialState.levels[0].strobeMode =
1764                         cypress_get_strobe_mode_settings(rdev,
1765                                                          initial_state->performance_levels[0].mclk);
1766
1767                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1768                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1769                 else
1770                         table->initialState.levels[0].mcFlags =  0;
1771         }
1772
1773         table->initialState.levelCount = 1;
1774
1775         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1776
1777         table->initialState.levels[0].dpm2.MaxPS = 0;
1778         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1779         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1780         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1781
1782         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1783         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1784
1785         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1786         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1787
1788         return 0;
1789 }
1790
1791 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1792                                       NISLANDS_SMC_STATETABLE *table)
1793 {
1794         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1795         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1796         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1797         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1798         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1799         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1800         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1801         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1802         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1803         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1804         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1805         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1806         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1807         u32 reg;
1808         int ret;
1809
1810         table->ACPIState = table->initialState;
1811
1812         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1813
1814         if (pi->acpi_vddc) {
1815                 ret = ni_populate_voltage_value(rdev,
1816                                                 &eg_pi->vddc_voltage_table,
1817                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1818                 if (!ret) {
1819                         u16 std_vddc;
1820
1821                         ret = ni_get_std_voltage_value(rdev,
1822                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1823                         if (!ret)
1824                                 ni_populate_std_voltage_value(rdev, std_vddc,
1825                                                               table->ACPIState.levels[0].vddc.index,
1826                                                               &table->ACPIState.levels[0].std_vddc);
1827                 }
1828
1829                 if (pi->pcie_gen2) {
1830                         if (pi->acpi_pcie_gen2)
1831                                 table->ACPIState.levels[0].gen2PCIE = 1;
1832                         else
1833                                 table->ACPIState.levels[0].gen2PCIE = 0;
1834                 } else {
1835                         table->ACPIState.levels[0].gen2PCIE = 0;
1836                 }
1837         } else {
1838                 ret = ni_populate_voltage_value(rdev,
1839                                                 &eg_pi->vddc_voltage_table,
1840                                                 pi->min_vddc_in_table,
1841                                                 &table->ACPIState.levels[0].vddc);
1842                 if (!ret) {
1843                         u16 std_vddc;
1844
1845                         ret = ni_get_std_voltage_value(rdev,
1846                                                        &table->ACPIState.levels[0].vddc,
1847                                                        &std_vddc);
1848                         if (!ret)
1849                                 ni_populate_std_voltage_value(rdev, std_vddc,
1850                                                               table->ACPIState.levels[0].vddc.index,
1851                                                               &table->ACPIState.levels[0].std_vddc);
1852                 }
1853                 table->ACPIState.levels[0].gen2PCIE = 0;
1854         }
1855
1856         if (eg_pi->acpi_vddci) {
1857                 if (eg_pi->vddci_control)
1858                         ni_populate_voltage_value(rdev,
1859                                                   &eg_pi->vddci_voltage_table,
1860                                                   eg_pi->acpi_vddci,
1861                                                   &table->ACPIState.levels[0].vddci);
1862         }
1863
1864
1865         mpll_ad_func_cntl &= ~PDNB;
1866
1867         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1868
1869         if (pi->mem_gddr5)
1870                 mpll_dq_func_cntl &= ~PDNB;
1871         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1872
1873
1874         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1875                              MRDCKA1_RESET |
1876                              MRDCKB0_RESET |
1877                              MRDCKB1_RESET |
1878                              MRDCKC0_RESET |
1879                              MRDCKC1_RESET |
1880                              MRDCKD0_RESET |
1881                              MRDCKD1_RESET);
1882
1883         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1884                               MRDCKA1_PDNB |
1885                               MRDCKB0_PDNB |
1886                               MRDCKB1_PDNB |
1887                               MRDCKC0_PDNB |
1888                               MRDCKC1_PDNB |
1889                               MRDCKD0_PDNB |
1890                               MRDCKD1_PDNB);
1891
1892         dll_cntl |= (MRDCKA0_BYPASS |
1893                      MRDCKA1_BYPASS |
1894                      MRDCKB0_BYPASS |
1895                      MRDCKB1_BYPASS |
1896                      MRDCKC0_BYPASS |
1897                      MRDCKC1_BYPASS |
1898                      MRDCKD0_BYPASS |
1899                      MRDCKD1_BYPASS);
1900
1901         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1902         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1903
1904         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1905         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1906         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1907         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1908         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1909         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1910
1911         table->ACPIState.levels[0].mclk.mclk_value = 0;
1912
1913         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1914         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1915         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1916         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1917
1918         table->ACPIState.levels[0].sclk.sclk_value = 0;
1919
1920         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1921
1922         if (eg_pi->dynamic_ac_timing)
1923                 table->ACPIState.levels[0].ACIndex = 1;
1924
1925         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1926         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1927         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1928         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1929
1930         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1931         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1932
1933         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1934         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1935
1936         return 0;
1937 }
1938
1939 static int ni_init_smc_table(struct radeon_device *rdev)
1940 {
1941         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1942         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1943         int ret;
1944         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1945         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1946
1947         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1948
1949         ni_populate_smc_voltage_tables(rdev, table);
1950
1951         switch (rdev->pm.int_thermal_type) {
1952         case THERMAL_TYPE_NI:
1953         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1954                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1955                 break;
1956         case THERMAL_TYPE_NONE:
1957                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1958                 break;
1959         default:
1960                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1961                 break;
1962         }
1963
1964         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1965                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1966
1967         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1968                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1969
1970         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1971                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1972
1973         if (pi->mem_gddr5)
1974                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1975
1976         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1977         if (ret)
1978                 return ret;
1979
1980         ret = ni_populate_smc_acpi_state(rdev, table);
1981         if (ret)
1982                 return ret;
1983
1984         table->driverState = table->initialState;
1985
1986         table->ULVState = table->initialState;
1987
1988         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1989                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
1990         if (ret)
1991                 return ret;
1992
1993         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1994                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1995 }
1996
1997 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1998                                     u32 engine_clock,
1999                                     NISLANDS_SMC_SCLK_VALUE *sclk)
2000 {
2001         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2002         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2003         struct atom_clock_dividers dividers;
2004         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2005         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2006         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2007         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2008         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2009         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2010         u64 tmp;
2011         u32 reference_clock = rdev->clock.spll.reference_freq;
2012         u32 reference_divider;
2013         u32 fbdiv;
2014         int ret;
2015
2016         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2017                                              engine_clock, false, &dividers);
2018         if (ret)
2019                 return ret;
2020
2021         reference_divider = 1 + dividers.ref_div;
2022
2023
2024         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2025         do_div(tmp, reference_clock);
2026         fbdiv = (u32) tmp;
2027
2028         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2029         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2030         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2031
2032         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2033         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2034
2035         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2036         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2037         spll_func_cntl_3 |= SPLL_DITHEN;
2038
2039         if (pi->sclk_ss) {
2040                 struct radeon_atom_ss ss;
2041                 u32 vco_freq = engine_clock * dividers.post_div;
2042
2043                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2044                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2045                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2046                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2047
2048                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2049                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2050                         cg_spll_spread_spectrum |= SSEN;
2051
2052                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2053                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2054                 }
2055         }
2056
2057         sclk->sclk_value = engine_clock;
2058         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2059         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2060         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2061         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2062         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2063         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2064
2065         return 0;
2066 }
2067
2068 static int ni_populate_sclk_value(struct radeon_device *rdev,
2069                                   u32 engine_clock,
2070                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2071 {
2072         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2073         int ret;
2074
2075         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2076         if (!ret) {
2077                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2078                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2079                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2080                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2081                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2082                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2083                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2084         }
2085
2086         return ret;
2087 }
2088
2089 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2090 {
2091         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2092         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2093         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2094         NISLANDS_SMC_SCLK_VALUE sclk_params;
2095         u32 fb_div;
2096         u32 p_div;
2097         u32 clk_s;
2098         u32 clk_v;
2099         u32 sclk = 0;
2100         int i, ret;
2101         u32 tmp;
2102
2103         if (ni_pi->spll_table_start == 0)
2104                 return -EINVAL;
2105
2106         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2107         if (spll_table == NULL)
2108                 return -ENOMEM;
2109
2110         for (i = 0; i < 256; i++) {
2111                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2112                 if (ret)
2113                         break;
2114
2115                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2116                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2117                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2118                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2119
2120                 fb_div &= ~0x00001FFF;
2121                 fb_div >>= 1;
2122                 clk_v >>= 6;
2123
2124                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2125                         ret = -EINVAL;
2126
2127                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2128                         ret = -EINVAL;
2129
2130                 if (fb_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT))
2131                         ret = -EINVAL;
2132
2133                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2134                         ret = -EINVAL;
2135
2136                 if (ret)
2137                         break;
2138
2139                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2140                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2141                 spll_table->freq[i] = cpu_to_be32(tmp);
2142
2143                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2144                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2145                 spll_table->ss[i] = cpu_to_be32(tmp);
2146
2147                 sclk += 512;
2148         }
2149
2150         if (!ret)
2151                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2152                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2153
2154         kfree(spll_table);
2155
2156         return ret;
2157 }
2158
2159 static int ni_populate_mclk_value(struct radeon_device *rdev,
2160                                   u32 engine_clock,
2161                                   u32 memory_clock,
2162                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2163                                   bool strobe_mode,
2164                                   bool dll_state_on)
2165 {
2166         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2167         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2168         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2169         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2170         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2171         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2172         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2173         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2174         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2175         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2176         struct atom_clock_dividers dividers;
2177         u32 ibias;
2178         u32 dll_speed;
2179         int ret;
2180         u32 mc_seq_misc7;
2181
2182         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2183                                              memory_clock, strobe_mode, &dividers);
2184         if (ret)
2185                 return ret;
2186
2187         if (!strobe_mode) {
2188                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2189
2190                 if (mc_seq_misc7 & 0x8000000)
2191                         dividers.post_div = 1;
2192         }
2193
2194         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2195
2196         mpll_ad_func_cntl &= ~(CLKR_MASK |
2197                                YCLK_POST_DIV_MASK |
2198                                CLKF_MASK |
2199                                CLKFRAC_MASK |
2200                                IBIAS_MASK);
2201         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2202         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2203         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2204         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2205         mpll_ad_func_cntl |= IBIAS(ibias);
2206
2207         if (dividers.vco_mode)
2208                 mpll_ad_func_cntl_2 |= VCO_MODE;
2209         else
2210                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2211
2212         if (pi->mem_gddr5) {
2213                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2214                                        YCLK_POST_DIV_MASK |
2215                                        CLKF_MASK |
2216                                        CLKFRAC_MASK |
2217                                        IBIAS_MASK);
2218                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2219                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2220                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2221                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2222                 mpll_dq_func_cntl |= IBIAS(ibias);
2223
2224                 if (strobe_mode)
2225                         mpll_dq_func_cntl &= ~PDNB;
2226                 else
2227                         mpll_dq_func_cntl |= PDNB;
2228
2229                 if (dividers.vco_mode)
2230                         mpll_dq_func_cntl_2 |= VCO_MODE;
2231                 else
2232                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2233         }
2234
2235         if (pi->mclk_ss) {
2236                 struct radeon_atom_ss ss;
2237                 u32 vco_freq = memory_clock * dividers.post_div;
2238
2239                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2240                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2241                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2242                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2243                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2244                         u32 clk_v = ss.percentage *
2245                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2246
2247                         mpll_ss1 &= ~CLKV_MASK;
2248                         mpll_ss1 |= CLKV(clk_v);
2249
2250                         mpll_ss2 &= ~CLKS_MASK;
2251                         mpll_ss2 |= CLKS(clk_s);
2252                 }
2253         }
2254
2255         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2256                                         memory_clock);
2257
2258         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2259         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2260         if (dll_state_on)
2261                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2262                                      MRDCKA1_PDNB |
2263                                      MRDCKB0_PDNB |
2264                                      MRDCKB1_PDNB |
2265                                      MRDCKC0_PDNB |
2266                                      MRDCKC1_PDNB |
2267                                      MRDCKD0_PDNB |
2268                                      MRDCKD1_PDNB);
2269         else
2270                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2271                                       MRDCKA1_PDNB |
2272                                       MRDCKB0_PDNB |
2273                                       MRDCKB1_PDNB |
2274                                       MRDCKC0_PDNB |
2275                                       MRDCKC1_PDNB |
2276                                       MRDCKD0_PDNB |
2277                                       MRDCKD1_PDNB);
2278
2279
2280         mclk->mclk_value = cpu_to_be32(memory_clock);
2281         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2282         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2283         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2284         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2285         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2286         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2287         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2288         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2289
2290         return 0;
2291 }
2292
2293 static void ni_populate_smc_sp(struct radeon_device *rdev,
2294                                struct radeon_ps *radeon_state,
2295                                NISLANDS_SMC_SWSTATE *smc_state)
2296 {
2297         struct ni_ps *ps = ni_get_ps(radeon_state);
2298         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2299         int i;
2300
2301         for (i = 0; i < ps->performance_level_count - 1; i++)
2302                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2303
2304         smc_state->levels[ps->performance_level_count - 1].bSP =
2305                 cpu_to_be32(pi->psp);
2306 }
2307
2308 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2309                                          struct rv7xx_pl *pl,
2310                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2311 {
2312         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2313         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2314         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2315         int ret;
2316         bool dll_state_on;
2317         u16 std_vddc;
2318         u32 tmp = RREG32(DC_STUTTER_CNTL);
2319
2320         level->gen2PCIE = pi->pcie_gen2 ?
2321                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2322
2323         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2324         if (ret)
2325                 return ret;
2326
2327         level->mcFlags =  0;
2328         if (pi->mclk_stutter_mode_threshold &&
2329             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2330             !eg_pi->uvd_enabled &&
2331             (tmp & DC_STUTTER_ENABLE_A) &&
2332             (tmp & DC_STUTTER_ENABLE_B))
2333                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2334
2335         if (pi->mem_gddr5) {
2336                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2337                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2338                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2339                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2340
2341                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2342
2343                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2344                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2345                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2346                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2347                         else
2348                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2349                 } else {
2350                         dll_state_on = false;
2351                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2352                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2353                 }
2354
2355                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2356                                              &level->mclk,
2357                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2358                                              dll_state_on);
2359         } else
2360                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2361
2362         if (ret)
2363                 return ret;
2364
2365         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2366                                         pl->vddc, &level->vddc);
2367         if (ret)
2368                 return ret;
2369
2370         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2371         if (ret)
2372                 return ret;
2373
2374         ni_populate_std_voltage_value(rdev, std_vddc,
2375                                       level->vddc.index, &level->std_vddc);
2376
2377         if (eg_pi->vddci_control) {
2378                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2379                                                 pl->vddci, &level->vddci);
2380                 if (ret)
2381                         return ret;
2382         }
2383
2384         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2385
2386         return ret;
2387 }
2388
2389 static int ni_populate_smc_t(struct radeon_device *rdev,
2390                              struct radeon_ps *radeon_state,
2391                              NISLANDS_SMC_SWSTATE *smc_state)
2392 {
2393         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2394         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2395         struct ni_ps *state = ni_get_ps(radeon_state);
2396         u32 a_t;
2397         u32 t_l, t_h;
2398         u32 high_bsp;
2399         int i, ret;
2400
2401         if (state->performance_level_count >= 9)
2402                 return -EINVAL;
2403
2404         if (state->performance_level_count < 2) {
2405                 a_t = CG_R(0xffff) | CG_L(0);
2406                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2407                 return 0;
2408         }
2409
2410         smc_state->levels[0].aT = cpu_to_be32(0);
2411
2412         for (i = 0; i <= state->performance_level_count - 2; i++) {
2413                 if (eg_pi->uvd_enabled)
2414                         ret = r600_calculate_at(
2415                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2416                                 100 * R600_AH_DFLT,
2417                                 state->performance_levels[i + 1].sclk,
2418                                 state->performance_levels[i].sclk,
2419                                 &t_l,
2420                                 &t_h);
2421                 else
2422                         ret = r600_calculate_at(
2423                                 1000 * (i + 1),
2424                                 100 * R600_AH_DFLT,
2425                                 state->performance_levels[i + 1].sclk,
2426                                 state->performance_levels[i].sclk,
2427                                 &t_l,
2428                                 &t_h);
2429
2430                 if (ret) {
2431                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2432                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2433                 }
2434
2435                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2436                 a_t |= CG_R(t_l * pi->bsp / 20000);
2437                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2438
2439                 high_bsp = (i == state->performance_level_count - 2) ?
2440                         pi->pbsp : pi->bsp;
2441
2442                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2443                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2444         }
2445
2446         return 0;
2447 }
2448
2449 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2450                                                 struct radeon_ps *radeon_state,
2451                                                 NISLANDS_SMC_SWSTATE *smc_state)
2452 {
2453         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2454         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2455         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2456         struct ni_ps *state = ni_get_ps(radeon_state);
2457         u32 prev_sclk;
2458         u32 max_sclk;
2459         u32 min_sclk;
2460         int i, ret;
2461         u32 tdp_limit;
2462         u32 near_tdp_limit;
2463         u32 power_boost_limit;
2464         u8 max_ps_percent;
2465
2466         if (ni_pi->enable_power_containment == false)
2467                 return 0;
2468
2469         if (state->performance_level_count == 0)
2470                 return -EINVAL;
2471
2472         if (smc_state->levelCount != state->performance_level_count)
2473                 return -EINVAL;
2474
2475         ret = ni_calculate_adjusted_tdp_limits(rdev,
2476                                                false, /* ??? */
2477                                                rdev->pm.dpm.tdp_adjustment,
2478                                                &tdp_limit,
2479                                                &near_tdp_limit);
2480         if (ret)
2481                 return ret;
2482
2483         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2484
2485         ret = rv770_write_smc_sram_dword(rdev,
2486                                          pi->state_table_start +
2487                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2488                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2489                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2490                                          pi->sram_end);
2491         if (ret)
2492                 power_boost_limit = 0;
2493
2494         smc_state->levels[0].dpm2.MaxPS = 0;
2495         smc_state->levels[0].dpm2.NearTDPDec = 0;
2496         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2497         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2498         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2499
2500         for (i = 1; i < state->performance_level_count; i++) {
2501                 prev_sclk = state->performance_levels[i-1].sclk;
2502                 max_sclk  = state->performance_levels[i].sclk;
2503                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2504                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2505
2506                 if (max_sclk < prev_sclk)
2507                         return -EINVAL;
2508
2509                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2510                         min_sclk = max_sclk;
2511                 else if (1 == i)
2512                         min_sclk = prev_sclk;
2513                 else
2514                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2515
2516                 if (min_sclk < state->performance_levels[0].sclk)
2517                         min_sclk = state->performance_levels[0].sclk;
2518
2519                 if (min_sclk == 0)
2520                         return -EINVAL;
2521
2522                 smc_state->levels[i].dpm2.MaxPS =
2523                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2524                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2525                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2526                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2527                 smc_state->levels[i].stateFlags |=
2528                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2529                         PPSMC_STATEFLAG_POWERBOOST : 0;
2530         }
2531
2532         return 0;
2533 }
2534
2535 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2536                                          struct radeon_ps *radeon_state,
2537                                          NISLANDS_SMC_SWSTATE *smc_state)
2538 {
2539         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2540         struct ni_ps *state = ni_get_ps(radeon_state);
2541         u32 sq_power_throttle;
2542         u32 sq_power_throttle2;
2543         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2544         int i;
2545
2546         if (state->performance_level_count == 0)
2547                 return -EINVAL;
2548
2549         if (smc_state->levelCount != state->performance_level_count)
2550                 return -EINVAL;
2551
2552         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2553                 return -EINVAL;
2554
2555         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2556                 enable_sq_ramping = false;
2557
2558         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2559                 enable_sq_ramping = false;
2560
2561         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2562                 enable_sq_ramping = false;
2563
2564         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2565                 enable_sq_ramping = false;
2566
2567         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2568                 enable_sq_ramping = false;
2569
2570         for (i = 0; i < state->performance_level_count; i++) {
2571                 sq_power_throttle  = 0;
2572                 sq_power_throttle2 = 0;
2573
2574                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2575                     enable_sq_ramping) {
2576                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2577                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2578                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2579                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2580                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2581                 } else {
2582                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2583                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2584                 }
2585
2586                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2587                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2588         }
2589
2590         return 0;
2591 }
2592
2593 static int ni_enable_power_containment(struct radeon_device *rdev,
2594                                        struct radeon_ps *radeon_new_state,
2595                                        bool enable)
2596 {
2597         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2598         PPSMC_Result smc_result;
2599         int ret = 0;
2600
2601         if (ni_pi->enable_power_containment) {
2602                 if (enable) {
2603                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2604                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2605                                 if (smc_result != PPSMC_Result_OK) {
2606                                         ret = -EINVAL;
2607                                         ni_pi->pc_enabled = false;
2608                                 } else {
2609                                         ni_pi->pc_enabled = true;
2610                                 }
2611                         }
2612                 } else {
2613                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2614                         if (smc_result != PPSMC_Result_OK)
2615                                 ret = -EINVAL;
2616                         ni_pi->pc_enabled = false;
2617                 }
2618         }
2619
2620         return ret;
2621 }
2622
2623 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2624                                          struct radeon_ps *radeon_state,
2625                                          NISLANDS_SMC_SWSTATE *smc_state)
2626 {
2627         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2628         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2629         struct ni_ps *state = ni_get_ps(radeon_state);
2630         int i, ret;
2631         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2632
2633         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2634                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2635
2636         smc_state->levelCount = 0;
2637
2638         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2639                 return -EINVAL;
2640
2641         for (i = 0; i < state->performance_level_count; i++) {
2642                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2643                                                     &smc_state->levels[i]);
2644                 smc_state->levels[i].arbRefreshState =
2645                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2646
2647                 if (ret)
2648                         return ret;
2649
2650                 if (ni_pi->enable_power_containment)
2651                         smc_state->levels[i].displayWatermark =
2652                                 (state->performance_levels[i].sclk < threshold) ?
2653                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2654                 else
2655                         smc_state->levels[i].displayWatermark = (i < 2) ?
2656                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2657
2658                 if (eg_pi->dynamic_ac_timing)
2659                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2660                 else
2661                         smc_state->levels[i].ACIndex = 0;
2662
2663                 smc_state->levelCount++;
2664         }
2665
2666         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2667                                       cpu_to_be32(threshold / 512));
2668
2669         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2670
2671         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2672         if (ret)
2673                 ni_pi->enable_power_containment = false;
2674
2675         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2676         if (ret)
2677                 ni_pi->enable_sq_ramping = false;
2678
2679         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2680 }
2681
2682 static int ni_upload_sw_state(struct radeon_device *rdev,
2683                               struct radeon_ps *radeon_new_state)
2684 {
2685         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2686         u16 address = pi->state_table_start +
2687                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2688         NISLANDS_SMC_SWSTATE *smc_state;
2689         size_t state_size = struct_size(smc_state, levels,
2690                         NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE);
2691         int ret;
2692
2693         smc_state = kzalloc(state_size, GFP_KERNEL);
2694         if (smc_state == NULL)
2695                 return -ENOMEM;
2696
2697         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2698         if (ret)
2699                 goto done;
2700
2701         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2702
2703 done:
2704         kfree(smc_state);
2705
2706         return ret;
2707 }
2708
2709 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2710                                        struct ni_mc_reg_table *table)
2711 {
2712         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2713         u8 i, j, k;
2714         u32 temp_reg;
2715
2716         for (i = 0, j = table->last; i < table->last; i++) {
2717                 switch (table->mc_reg_address[i].s1) {
2718                 case MC_SEQ_MISC1 >> 2:
2719                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2720                                 return -EINVAL;
2721                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2722                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2723                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2724                         for (k = 0; k < table->num_entries; k++)
2725                                 table->mc_reg_table_entry[k].mc_data[j] =
2726                                         ((temp_reg & 0xffff0000)) |
2727                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2728                         j++;
2729                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2730                                 return -EINVAL;
2731
2732                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2733                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2734                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2735                         for(k = 0; k < table->num_entries; k++) {
2736                                 table->mc_reg_table_entry[k].mc_data[j] =
2737                                         (temp_reg & 0xffff0000) |
2738                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2739                                 if (!pi->mem_gddr5)
2740                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2741                         }
2742                         j++;
2743                         break;
2744                 case MC_SEQ_RESERVE_M >> 2:
2745                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2746                                 return -EINVAL;
2747                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2748                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2749                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2750                         for (k = 0; k < table->num_entries; k++)
2751                                 table->mc_reg_table_entry[k].mc_data[j] =
2752                                         (temp_reg & 0xffff0000) |
2753                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2754                         j++;
2755                         break;
2756                 default:
2757                         break;
2758                 }
2759         }
2760
2761         table->last = j;
2762
2763         return 0;
2764 }
2765
2766 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2767 {
2768         bool result = true;
2769
2770         switch (in_reg) {
2771         case  MC_SEQ_RAS_TIMING >> 2:
2772                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2773                 break;
2774         case MC_SEQ_CAS_TIMING >> 2:
2775                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2776                 break;
2777         case MC_SEQ_MISC_TIMING >> 2:
2778                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2779                 break;
2780         case MC_SEQ_MISC_TIMING2 >> 2:
2781                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2782                 break;
2783         case MC_SEQ_RD_CTL_D0 >> 2:
2784                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2785                 break;
2786         case MC_SEQ_RD_CTL_D1 >> 2:
2787                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2788                 break;
2789         case MC_SEQ_WR_CTL_D0 >> 2:
2790                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2791                 break;
2792         case MC_SEQ_WR_CTL_D1 >> 2:
2793                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2794                 break;
2795         case MC_PMG_CMD_EMRS >> 2:
2796                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2797                 break;
2798         case MC_PMG_CMD_MRS >> 2:
2799                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2800                 break;
2801         case MC_PMG_CMD_MRS1 >> 2:
2802                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2803                 break;
2804         case MC_SEQ_PMG_TIMING >> 2:
2805                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2806                 break;
2807         case MC_PMG_CMD_MRS2 >> 2:
2808                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2809                 break;
2810         default:
2811                 result = false;
2812                 break;
2813         }
2814
2815         return result;
2816 }
2817
2818 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2819 {
2820         u8 i, j;
2821
2822         for (i = 0; i < table->last; i++) {
2823                 for (j = 1; j < table->num_entries; j++) {
2824                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2825                                 table->valid_flag |= 1 << i;
2826                                 break;
2827                         }
2828                 }
2829         }
2830 }
2831
2832 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2833 {
2834         u32 i;
2835         u16 address;
2836
2837         for (i = 0; i < table->last; i++)
2838                 table->mc_reg_address[i].s0 =
2839                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2840                         address : table->mc_reg_address[i].s1;
2841 }
2842
2843 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2844                                       struct ni_mc_reg_table *ni_table)
2845 {
2846         u8 i, j;
2847
2848         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2849                 return -EINVAL;
2850         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2851                 return -EINVAL;
2852
2853         for (i = 0; i < table->last; i++)
2854                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2855         ni_table->last = table->last;
2856
2857         for (i = 0; i < table->num_entries; i++) {
2858                 ni_table->mc_reg_table_entry[i].mclk_max =
2859                         table->mc_reg_table_entry[i].mclk_max;
2860                 for (j = 0; j < table->last; j++)
2861                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2862                                 table->mc_reg_table_entry[i].mc_data[j];
2863         }
2864         ni_table->num_entries = table->num_entries;
2865
2866         return 0;
2867 }
2868
2869 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2870 {
2871         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2872         int ret;
2873         struct atom_mc_reg_table *table;
2874         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2875         u8 module_index = rv770_get_memory_module_index(rdev);
2876
2877         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2878         if (!table)
2879                 return -ENOMEM;
2880
2881         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2882         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2883         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2884         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2885         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2886         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2887         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2888         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2889         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2890         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2891         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2892         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2893         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2894
2895         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2896
2897         if (ret)
2898                 goto init_mc_done;
2899
2900         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2901
2902         if (ret)
2903                 goto init_mc_done;
2904
2905         ni_set_s0_mc_reg_index(ni_table);
2906
2907         ret = ni_set_mc_special_registers(rdev, ni_table);
2908
2909         if (ret)
2910                 goto init_mc_done;
2911
2912         ni_set_valid_flag(ni_table);
2913
2914 init_mc_done:
2915         kfree(table);
2916
2917         return ret;
2918 }
2919
2920 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2921                                          SMC_NIslands_MCRegisters *mc_reg_table)
2922 {
2923         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2924         u32 i, j;
2925
2926         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2927                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2928                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2929                                 break;
2930                         mc_reg_table->address[i].s0 =
2931                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2932                         mc_reg_table->address[i].s1 =
2933                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2934                         i++;
2935                 }
2936         }
2937         mc_reg_table->last = (u8)i;
2938 }
2939
2940
2941 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2942                                     SMC_NIslands_MCRegisterSet *data,
2943                                     u32 num_entries, u32 valid_flag)
2944 {
2945         u32 i, j;
2946
2947         for (i = 0, j = 0; j < num_entries; j++) {
2948                 if (valid_flag & (1 << j)) {
2949                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2950                         i++;
2951                 }
2952         }
2953 }
2954
2955 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2956                                                  struct rv7xx_pl *pl,
2957                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2958 {
2959         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2960         u32 i = 0;
2961
2962         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2963                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2964                         break;
2965         }
2966
2967         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2968                 --i;
2969
2970         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2971                                 mc_reg_table_data,
2972                                 ni_pi->mc_reg_table.last,
2973                                 ni_pi->mc_reg_table.valid_flag);
2974 }
2975
2976 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2977                                            struct radeon_ps *radeon_state,
2978                                            SMC_NIslands_MCRegisters *mc_reg_table)
2979 {
2980         struct ni_ps *state = ni_get_ps(radeon_state);
2981         int i;
2982
2983         for (i = 0; i < state->performance_level_count; i++) {
2984                 ni_convert_mc_reg_table_entry_to_smc(rdev,
2985                                                      &state->performance_levels[i],
2986                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2987         }
2988 }
2989
2990 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2991                                     struct radeon_ps *radeon_boot_state)
2992 {
2993         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2994         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2995         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2996         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
2997         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
2998
2999         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3000
3001         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3002
3003         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3004
3005         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3006                                              &mc_reg_table->data[0]);
3007
3008         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3009                                 &mc_reg_table->data[1],
3010                                 ni_pi->mc_reg_table.last,
3011                                 ni_pi->mc_reg_table.valid_flag);
3012
3013         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3014
3015         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3016                                        (u8 *)mc_reg_table,
3017                                        sizeof(SMC_NIslands_MCRegisters),
3018                                        pi->sram_end);
3019 }
3020
3021 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3022                                   struct radeon_ps *radeon_new_state)
3023 {
3024         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3025         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3026         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3027         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3028         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3029         u16 address;
3030
3031         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3032
3033         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3034
3035         address = eg_pi->mc_reg_table_start +
3036                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3037
3038         return rv770_copy_bytes_to_smc(rdev, address,
3039                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3040                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3041                                        pi->sram_end);
3042 }
3043
3044 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3045                                                    PP_NIslands_CACTABLES *cac_tables)
3046 {
3047         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3048         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3049         u32 leakage = 0;
3050         unsigned int i, j, table_size;
3051         s32 t;
3052         u32 smc_leakage, max_leakage = 0;
3053         u32 scaling_factor;
3054
3055         table_size = eg_pi->vddc_voltage_table.count;
3056
3057         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3058                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3059
3060         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3061
3062         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3063                 for (j = 0; j < table_size; j++) {
3064                         t = (1000 * ((i + 1) * 8));
3065
3066                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3067                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3068
3069                         ni_calculate_leakage_for_v_and_t(rdev,
3070                                                          &ni_pi->cac_data.leakage_coefficients,
3071                                                          eg_pi->vddc_voltage_table.entries[j].value,
3072                                                          t,
3073                                                          ni_pi->cac_data.i_leakage,
3074                                                          &leakage);
3075
3076                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3077                         if (smc_leakage > max_leakage)
3078                                 max_leakage = smc_leakage;
3079
3080                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3081                 }
3082         }
3083
3084         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3085                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3086                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3087         }
3088         return 0;
3089 }
3090
3091 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3092                                             PP_NIslands_CACTABLES *cac_tables)
3093 {
3094         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3095         struct radeon_cac_leakage_table *leakage_table =
3096                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3097         u32 i, j, table_size;
3098         u32 smc_leakage, max_leakage = 0;
3099         u32 scaling_factor;
3100
3101         if (!leakage_table)
3102                 return -EINVAL;
3103
3104         table_size = leakage_table->count;
3105
3106         if (eg_pi->vddc_voltage_table.count != table_size)
3107                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3108                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3109
3110         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3111                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3112
3113         if (table_size == 0)
3114                 return -EINVAL;
3115
3116         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3117
3118         for (j = 0; j < table_size; j++) {
3119                 smc_leakage = leakage_table->entries[j].leakage;
3120
3121                 if (smc_leakage > max_leakage)
3122                         max_leakage = smc_leakage;
3123
3124                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3125                         cac_tables->cac_lkge_lut[i][j] =
3126                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3127         }
3128
3129         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3130                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3131                         cac_tables->cac_lkge_lut[i][j] =
3132                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3133         }
3134         return 0;
3135 }
3136
3137 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3138 {
3139         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3140         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3141         PP_NIslands_CACTABLES *cac_tables = NULL;
3142         int i, ret;
3143         u32 reg;
3144
3145         if (ni_pi->enable_cac == false)
3146                 return 0;
3147
3148         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3149         if (!cac_tables)
3150                 return -ENOMEM;
3151
3152         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3153         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3154                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3155         WREG32(CG_CAC_CTRL, reg);
3156
3157         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3158                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3159
3160         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3161                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3162
3163         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3164         ni_pi->cac_data.pwr_const = 0;
3165         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3166         ni_pi->cac_data.bif_cac_value = 0;
3167         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3168         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3169         ni_pi->cac_data.allow_ovrflw = 0;
3170         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3171         ni_pi->cac_data.num_win_tdp = 0;
3172         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3173
3174         if (ni_pi->driver_calculate_cac_leakage)
3175                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3176         else
3177                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3178
3179         if (ret)
3180                 goto done_free;
3181
3182         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3183         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3184         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3185         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3186         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3187         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3188         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3189         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3190         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3191
3192         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3193                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3194
3195 done_free:
3196         if (ret) {
3197                 ni_pi->enable_cac = false;
3198                 ni_pi->enable_power_containment = false;
3199         }
3200
3201         kfree(cac_tables);
3202
3203         return 0;
3204 }
3205
3206 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3207 {
3208         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3209         u32 reg;
3210
3211         if (!ni_pi->enable_cac ||
3212             !ni_pi->cac_configuration_required)
3213                 return 0;
3214
3215         if (ni_pi->cac_weights == NULL)
3216                 return -EINVAL;
3217
3218         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3219                                                       WEIGHT_TCP_SIG1_MASK |
3220                                                       WEIGHT_TA_SIG_MASK);
3221         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3222                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3223                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3224         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3225
3226         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3227                                                       WEIGHT_TCC_EN1_MASK |
3228                                                       WEIGHT_TCC_EN2_MASK);
3229         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3230                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3231                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3232         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3233
3234         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3235                                                       WEIGHT_CB_EN1_MASK |
3236                                                       WEIGHT_CB_EN2_MASK |
3237                                                       WEIGHT_CB_EN3_MASK);
3238         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3239                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3240                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3241                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3242         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3243
3244         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3245                                                       WEIGHT_DB_SIG1_MASK |
3246                                                       WEIGHT_DB_SIG2_MASK |
3247                                                       WEIGHT_DB_SIG3_MASK);
3248         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3249                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3250                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3251                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3252         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3253
3254         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3255                                                       WEIGHT_SXM_SIG1_MASK |
3256                                                       WEIGHT_SXM_SIG2_MASK |
3257                                                       WEIGHT_SXS_SIG0_MASK |
3258                                                       WEIGHT_SXS_SIG1_MASK);
3259         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3260                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3261                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3262                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3263                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3264         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3265
3266         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3267                                                       WEIGHT_XBR_1_MASK |
3268                                                       WEIGHT_XBR_2_MASK |
3269                                                       WEIGHT_SPI_SIG0_MASK);
3270         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3271                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3272                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3273                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3274         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3275
3276         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3277                                                       WEIGHT_SPI_SIG2_MASK |
3278                                                       WEIGHT_SPI_SIG3_MASK |
3279                                                       WEIGHT_SPI_SIG4_MASK |
3280                                                       WEIGHT_SPI_SIG5_MASK);
3281         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3282                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3283                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3284                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3285                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3286         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3287
3288         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3289                                                       WEIGHT_LDS_SIG1_MASK |
3290                                                       WEIGHT_SC_MASK);
3291         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3292                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3293                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3294         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3295
3296         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3297                                                       WEIGHT_CP_MASK |
3298                                                       WEIGHT_PA_SIG0_MASK |
3299                                                       WEIGHT_PA_SIG1_MASK |
3300                                                       WEIGHT_VGT_SIG0_MASK);
3301         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3302                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3303                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3304                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3305                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3306         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3307
3308         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3309                                                       WEIGHT_VGT_SIG2_MASK |
3310                                                       WEIGHT_DC_SIG0_MASK |
3311                                                       WEIGHT_DC_SIG1_MASK |
3312                                                       WEIGHT_DC_SIG2_MASK);
3313         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3314                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3315                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3316                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3317                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3318         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3319
3320         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3321                                                       WEIGHT_UVD_SIG0_MASK |
3322                                                       WEIGHT_UVD_SIG1_MASK |
3323                                                       WEIGHT_SPARE0_MASK |
3324                                                       WEIGHT_SPARE1_MASK);
3325         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3326                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3327                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3328                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3329                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3330         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3331
3332         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3333                                                       WEIGHT_SQ_VSP0_MASK);
3334         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3335                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3336         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3337
3338         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3339         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3340         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3341
3342         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3343                                                         OVR_VAL_SPARE_0_MASK |
3344                                                         OVR_MODE_SPARE_1_MASK |
3345                                                         OVR_VAL_SPARE_1_MASK);
3346         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3347                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3348                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3349                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3350         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3351
3352         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3353                                            VSP0_MASK |
3354                                            GPR_MASK);
3355         reg |= (VSP(ni_pi->cac_weights->vsp) |
3356                 VSP0(ni_pi->cac_weights->vsp0) |
3357                 GPR(ni_pi->cac_weights->gpr));
3358         WREG32(SQ_CAC_THRESHOLD, reg);
3359
3360         reg = (MCDW_WR_ENABLE |
3361                MCDX_WR_ENABLE |
3362                MCDY_WR_ENABLE |
3363                MCDZ_WR_ENABLE |
3364                INDEX(0x09D4));
3365         WREG32(MC_CG_CONFIG, reg);
3366
3367         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3368                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3369                ALLOW_OVERFLOW);
3370         WREG32(MC_CG_DATAPORT, reg);
3371
3372         return 0;
3373 }
3374
3375 static int ni_enable_smc_cac(struct radeon_device *rdev,
3376                              struct radeon_ps *radeon_new_state,
3377                              bool enable)
3378 {
3379         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3380         int ret = 0;
3381         PPSMC_Result smc_result;
3382
3383         if (ni_pi->enable_cac) {
3384                 if (enable) {
3385                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3386                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3387
3388                                 if (ni_pi->support_cac_long_term_average) {
3389                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3390                                         if (PPSMC_Result_OK != smc_result)
3391                                                 ni_pi->support_cac_long_term_average = false;
3392                                 }
3393
3394                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3395                                 if (PPSMC_Result_OK != smc_result)
3396                                         ret = -EINVAL;
3397
3398                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3399                         }
3400                 } else if (ni_pi->cac_enabled) {
3401                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3402
3403                         ni_pi->cac_enabled = false;
3404
3405                         if (ni_pi->support_cac_long_term_average) {
3406                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3407                                 if (PPSMC_Result_OK != smc_result)
3408                                         ni_pi->support_cac_long_term_average = false;
3409                         }
3410                 }
3411         }
3412
3413         return ret;
3414 }
3415
3416 static int ni_pcie_performance_request(struct radeon_device *rdev,
3417                                        u8 perf_req, bool advertise)
3418 {
3419 #if defined(CONFIG_ACPI)
3420         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3421
3422         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3423             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3424                 if (eg_pi->pcie_performance_request_registered == false)
3425                         radeon_acpi_pcie_notify_device_ready(rdev);
3426                 eg_pi->pcie_performance_request_registered = true;
3427                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3428         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3429                     eg_pi->pcie_performance_request_registered) {
3430                 eg_pi->pcie_performance_request_registered = false;
3431                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3432         }
3433 #endif
3434         return 0;
3435 }
3436
3437 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3438 {
3439         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3440         u32 tmp;
3441
3442         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3443
3444         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3445             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3446                 pi->pcie_gen2 = true;
3447         else
3448                 pi->pcie_gen2 = false;
3449
3450         if (!pi->pcie_gen2)
3451                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3452
3453         return 0;
3454 }
3455
3456 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3457                                             bool enable)
3458 {
3459         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3460         u32 tmp, bif;
3461
3462         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3463
3464         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3465             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3466                 if (enable) {
3467                         if (!pi->boot_in_gen2) {
3468                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3469                                 bif |= CG_CLIENT_REQ(0xd);
3470                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3471                         }
3472                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3473                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3474                         tmp |= LC_GEN2_EN_STRAP;
3475
3476                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3477                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3478                         udelay(10);
3479                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3480                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3481                 } else {
3482                         if (!pi->boot_in_gen2) {
3483                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3484                                 bif |= CG_CLIENT_REQ(0xd);
3485                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3486
3487                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3488                                 tmp &= ~LC_GEN2_EN_STRAP;
3489                         }
3490                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3491                 }
3492         }
3493 }
3494
3495 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3496                                         bool enable)
3497 {
3498         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3499
3500         if (enable)
3501                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3502         else
3503                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3504 }
3505
3506 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3507                                            struct radeon_ps *new_ps,
3508                                            struct radeon_ps *old_ps)
3509 {
3510         struct ni_ps *new_state = ni_get_ps(new_ps);
3511         struct ni_ps *current_state = ni_get_ps(old_ps);
3512
3513         if ((new_ps->vclk == old_ps->vclk) &&
3514             (new_ps->dclk == old_ps->dclk))
3515                 return;
3516
3517         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3518             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3519                 return;
3520
3521         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3522 }
3523
3524 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3525                                           struct radeon_ps *new_ps,
3526                                           struct radeon_ps *old_ps)
3527 {
3528         struct ni_ps *new_state = ni_get_ps(new_ps);
3529         struct ni_ps *current_state = ni_get_ps(old_ps);
3530
3531         if ((new_ps->vclk == old_ps->vclk) &&
3532             (new_ps->dclk == old_ps->dclk))
3533                 return;
3534
3535         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3536             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3537                 return;
3538
3539         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3540 }
3541
3542 void ni_dpm_setup_asic(struct radeon_device *rdev)
3543 {
3544         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3545         int r;
3546
3547         r = ni_mc_load_microcode(rdev);
3548         if (r)
3549                 DRM_ERROR("Failed to load MC firmware!\n");
3550         ni_read_clock_registers(rdev);
3551         btc_read_arb_registers(rdev);
3552         rv770_get_memory_type(rdev);
3553         if (eg_pi->pcie_performance_request)
3554                 ni_advertise_gen2_capability(rdev);
3555         rv770_get_pcie_gen2_status(rdev);
3556         rv770_enable_acpi_pm(rdev);
3557 }
3558
3559 void ni_update_current_ps(struct radeon_device *rdev,
3560                           struct radeon_ps *rps)
3561 {
3562         struct ni_ps *new_ps = ni_get_ps(rps);
3563         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3564         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3565
3566         eg_pi->current_rps = *rps;
3567         ni_pi->current_ps = *new_ps;
3568         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3569 }
3570
3571 void ni_update_requested_ps(struct radeon_device *rdev,
3572                             struct radeon_ps *rps)
3573 {
3574         struct ni_ps *new_ps = ni_get_ps(rps);
3575         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3576         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3577
3578         eg_pi->requested_rps = *rps;
3579         ni_pi->requested_ps = *new_ps;
3580         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3581 }
3582
3583 int ni_dpm_enable(struct radeon_device *rdev)
3584 {
3585         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3586         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3587         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3588         int ret;
3589
3590         if (pi->gfx_clock_gating)
3591                 ni_cg_clockgating_default(rdev);
3592         if (btc_dpm_enabled(rdev))
3593                 return -EINVAL;
3594         if (pi->mg_clock_gating)
3595                 ni_mg_clockgating_default(rdev);
3596         if (eg_pi->ls_clock_gating)
3597                 ni_ls_clockgating_default(rdev);
3598         if (pi->voltage_control) {
3599                 rv770_enable_voltage_control(rdev, true);
3600                 ret = cypress_construct_voltage_tables(rdev);
3601                 if (ret) {
3602                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3603                         return ret;
3604                 }
3605         }
3606         if (eg_pi->dynamic_ac_timing) {
3607                 ret = ni_initialize_mc_reg_table(rdev);
3608                 if (ret)
3609                         eg_pi->dynamic_ac_timing = false;
3610         }
3611         if (pi->dynamic_ss)
3612                 cypress_enable_spread_spectrum(rdev, true);
3613         if (pi->thermal_protection)
3614                 rv770_enable_thermal_protection(rdev, true);
3615         rv770_setup_bsp(rdev);
3616         rv770_program_git(rdev);
3617         rv770_program_tp(rdev);
3618         rv770_program_tpp(rdev);
3619         rv770_program_sstp(rdev);
3620         cypress_enable_display_gap(rdev);
3621         rv770_program_vc(rdev);
3622         if (pi->dynamic_pcie_gen2)
3623                 ni_enable_dynamic_pcie_gen2(rdev, true);
3624         ret = rv770_upload_firmware(rdev);
3625         if (ret) {
3626                 DRM_ERROR("rv770_upload_firmware failed\n");
3627                 return ret;
3628         }
3629         ret = ni_process_firmware_header(rdev);
3630         if (ret) {
3631                 DRM_ERROR("ni_process_firmware_header failed\n");
3632                 return ret;
3633         }
3634         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3635         if (ret) {
3636                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3637                 return ret;
3638         }
3639         ret = ni_init_smc_table(rdev);
3640         if (ret) {
3641                 DRM_ERROR("ni_init_smc_table failed\n");
3642                 return ret;
3643         }
3644         ret = ni_init_smc_spll_table(rdev);
3645         if (ret) {
3646                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3647                 return ret;
3648         }
3649         ret = ni_init_arb_table_index(rdev);
3650         if (ret) {
3651                 DRM_ERROR("ni_init_arb_table_index failed\n");
3652                 return ret;
3653         }
3654         if (eg_pi->dynamic_ac_timing) {
3655                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3656                 if (ret) {
3657                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3658                         return ret;
3659                 }
3660         }
3661         ret = ni_initialize_smc_cac_tables(rdev);
3662         if (ret) {
3663                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3664                 return ret;
3665         }
3666         ret = ni_initialize_hardware_cac_manager(rdev);
3667         if (ret) {
3668                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3669                 return ret;
3670         }
3671         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3672         if (ret) {
3673                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3674                 return ret;
3675         }
3676         ni_program_response_times(rdev);
3677         r7xx_start_smc(rdev);
3678         ret = cypress_notify_smc_display_change(rdev, false);
3679         if (ret) {
3680                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3681                 return ret;
3682         }
3683         cypress_enable_sclk_control(rdev, true);
3684         if (eg_pi->memory_transition)
3685                 cypress_enable_mclk_control(rdev, true);
3686         cypress_start_dpm(rdev);
3687         if (pi->gfx_clock_gating)
3688                 ni_gfx_clockgating_enable(rdev, true);
3689         if (pi->mg_clock_gating)
3690                 ni_mg_clockgating_enable(rdev, true);
3691         if (eg_pi->ls_clock_gating)
3692                 ni_ls_clockgating_enable(rdev, true);
3693
3694         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3695
3696         ni_update_current_ps(rdev, boot_ps);
3697
3698         return 0;
3699 }
3700
3701 void ni_dpm_disable(struct radeon_device *rdev)
3702 {
3703         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3704         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3705         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3706
3707         if (!btc_dpm_enabled(rdev))
3708                 return;
3709         rv770_clear_vc(rdev);
3710         if (pi->thermal_protection)
3711                 rv770_enable_thermal_protection(rdev, false);
3712         ni_enable_power_containment(rdev, boot_ps, false);
3713         ni_enable_smc_cac(rdev, boot_ps, false);
3714         cypress_enable_spread_spectrum(rdev, false);
3715         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3716         if (pi->dynamic_pcie_gen2)
3717                 ni_enable_dynamic_pcie_gen2(rdev, false);
3718
3719         if (rdev->irq.installed &&
3720             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3721                 rdev->irq.dpm_thermal = false;
3722                 radeon_irq_set(rdev);
3723         }
3724
3725         if (pi->gfx_clock_gating)
3726                 ni_gfx_clockgating_enable(rdev, false);
3727         if (pi->mg_clock_gating)
3728                 ni_mg_clockgating_enable(rdev, false);
3729         if (eg_pi->ls_clock_gating)
3730                 ni_ls_clockgating_enable(rdev, false);
3731         ni_stop_dpm(rdev);
3732         btc_reset_to_default(rdev);
3733         ni_stop_smc(rdev);
3734         ni_force_switch_to_arb_f0(rdev);
3735
3736         ni_update_current_ps(rdev, boot_ps);
3737 }
3738
3739 static int ni_power_control_set_level(struct radeon_device *rdev)
3740 {
3741         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3742         int ret;
3743
3744         ret = ni_restrict_performance_levels_before_switch(rdev);
3745         if (ret)
3746                 return ret;
3747         ret = rv770_halt_smc(rdev);
3748         if (ret)
3749                 return ret;
3750         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3751         if (ret)
3752                 return ret;
3753         ret = rv770_resume_smc(rdev);
3754         if (ret)
3755                 return ret;
3756         ret = rv770_set_sw_state(rdev);
3757         if (ret)
3758                 return ret;
3759
3760         return 0;
3761 }
3762
3763 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3764 {
3765         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3766         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3767         struct radeon_ps *new_ps = &requested_ps;
3768
3769         ni_update_requested_ps(rdev, new_ps);
3770
3771         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3772
3773         return 0;
3774 }
3775
3776 int ni_dpm_set_power_state(struct radeon_device *rdev)
3777 {
3778         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3779         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3780         struct radeon_ps *old_ps = &eg_pi->current_rps;
3781         int ret;
3782
3783         ret = ni_restrict_performance_levels_before_switch(rdev);
3784         if (ret) {
3785                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3786                 return ret;
3787         }
3788         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3789         ret = ni_enable_power_containment(rdev, new_ps, false);
3790         if (ret) {
3791                 DRM_ERROR("ni_enable_power_containment failed\n");
3792                 return ret;
3793         }
3794         ret = ni_enable_smc_cac(rdev, new_ps, false);
3795         if (ret) {
3796                 DRM_ERROR("ni_enable_smc_cac failed\n");
3797                 return ret;
3798         }
3799         ret = rv770_halt_smc(rdev);
3800         if (ret) {
3801                 DRM_ERROR("rv770_halt_smc failed\n");
3802                 return ret;
3803         }
3804         if (eg_pi->smu_uvd_hs)
3805                 btc_notify_uvd_to_smc(rdev, new_ps);
3806         ret = ni_upload_sw_state(rdev, new_ps);
3807         if (ret) {
3808                 DRM_ERROR("ni_upload_sw_state failed\n");
3809                 return ret;
3810         }
3811         if (eg_pi->dynamic_ac_timing) {
3812                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3813                 if (ret) {
3814                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3815                         return ret;
3816                 }
3817         }
3818         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3819         if (ret) {
3820                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3821                 return ret;
3822         }
3823         ret = rv770_resume_smc(rdev);
3824         if (ret) {
3825                 DRM_ERROR("rv770_resume_smc failed\n");
3826                 return ret;
3827         }
3828         ret = rv770_set_sw_state(rdev);
3829         if (ret) {
3830                 DRM_ERROR("rv770_set_sw_state failed\n");
3831                 return ret;
3832         }
3833         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3834         ret = ni_enable_smc_cac(rdev, new_ps, true);
3835         if (ret) {
3836                 DRM_ERROR("ni_enable_smc_cac failed\n");
3837                 return ret;
3838         }
3839         ret = ni_enable_power_containment(rdev, new_ps, true);
3840         if (ret) {
3841                 DRM_ERROR("ni_enable_power_containment failed\n");
3842                 return ret;
3843         }
3844
3845         /* update tdp */
3846         ret = ni_power_control_set_level(rdev);
3847         if (ret) {
3848                 DRM_ERROR("ni_power_control_set_level failed\n");
3849                 return ret;
3850         }
3851
3852         return 0;
3853 }
3854
3855 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3856 {
3857         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3858         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3859
3860         ni_update_current_ps(rdev, new_ps);
3861 }
3862
3863 #if 0
3864 void ni_dpm_reset_asic(struct radeon_device *rdev)
3865 {
3866         ni_restrict_performance_levels_before_switch(rdev);
3867         rv770_set_boot_state(rdev);
3868 }
3869 #endif
3870
3871 union power_info {
3872         struct _ATOM_POWERPLAY_INFO info;
3873         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3874         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3875         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3876         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3877         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3878 };
3879
3880 union pplib_clock_info {
3881         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3882         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3883         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3884         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3885 };
3886
3887 union pplib_power_state {
3888         struct _ATOM_PPLIB_STATE v1;
3889         struct _ATOM_PPLIB_STATE_V2 v2;
3890 };
3891
3892 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3893                                           struct radeon_ps *rps,
3894                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3895                                           u8 table_rev)
3896 {
3897         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3898         rps->class = le16_to_cpu(non_clock_info->usClassification);
3899         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3900
3901         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3902                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3903                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3904         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3905                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3906                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3907         } else {
3908                 rps->vclk = 0;
3909                 rps->dclk = 0;
3910         }
3911
3912         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3913                 rdev->pm.dpm.boot_ps = rps;
3914         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3915                 rdev->pm.dpm.uvd_ps = rps;
3916 }
3917
3918 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3919                                       struct radeon_ps *rps, int index,
3920                                       union pplib_clock_info *clock_info)
3921 {
3922         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3923         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3924         struct ni_ps *ps = ni_get_ps(rps);
3925         struct rv7xx_pl *pl = &ps->performance_levels[index];
3926
3927         ps->performance_level_count = index + 1;
3928
3929         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3930         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3931         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3932         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3933
3934         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3935         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3936         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3937
3938         /* patch up vddc if necessary */
3939         if (pl->vddc == 0xff01) {
3940                 if (pi->max_vddc)
3941                         pl->vddc = pi->max_vddc;
3942         }
3943
3944         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3945                 pi->acpi_vddc = pl->vddc;
3946                 eg_pi->acpi_vddci = pl->vddci;
3947                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3948                         pi->acpi_pcie_gen2 = true;
3949                 else
3950                         pi->acpi_pcie_gen2 = false;
3951         }
3952
3953         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3954                 eg_pi->ulv.supported = true;
3955                 eg_pi->ulv.pl = pl;
3956         }
3957
3958         if (pi->min_vddc_in_table > pl->vddc)
3959                 pi->min_vddc_in_table = pl->vddc;
3960
3961         if (pi->max_vddc_in_table < pl->vddc)
3962                 pi->max_vddc_in_table = pl->vddc;
3963
3964         /* patch up boot state */
3965         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3966                 u16 vddc, vddci, mvdd;
3967                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3968                 pl->mclk = rdev->clock.default_mclk;
3969                 pl->sclk = rdev->clock.default_sclk;
3970                 pl->vddc = vddc;
3971                 pl->vddci = vddci;
3972         }
3973
3974         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3975             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3976                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3977                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3978                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3979                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3980         }
3981 }
3982
3983 static int ni_parse_power_table(struct radeon_device *rdev)
3984 {
3985         struct radeon_mode_info *mode_info = &rdev->mode_info;
3986         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3987         union pplib_power_state *power_state;
3988         int i, j;
3989         union pplib_clock_info *clock_info;
3990         union power_info *power_info;
3991         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
3992         u16 data_offset;
3993         u8 frev, crev;
3994         struct ni_ps *ps;
3995
3996         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
3997                                    &frev, &crev, &data_offset))
3998                 return -EINVAL;
3999         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4000
4001         rdev->pm.dpm.ps = kcalloc(power_info->pplib.ucNumStates,
4002                                   sizeof(struct radeon_ps),
4003                                   GFP_KERNEL);
4004         if (!rdev->pm.dpm.ps)
4005                 return -ENOMEM;
4006
4007         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4008                 power_state = (union pplib_power_state *)
4009                         (mode_info->atom_context->bios + data_offset +
4010                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4011                          i * power_info->pplib.ucStateEntrySize);
4012                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4013                         (mode_info->atom_context->bios + data_offset +
4014                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4015                          (power_state->v1.ucNonClockStateIndex *
4016                           power_info->pplib.ucNonClockSize));
4017                 if (power_info->pplib.ucStateEntrySize - 1) {
4018                         u8 *idx;
4019                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4020                         if (ps == NULL) {
4021                                 kfree(rdev->pm.dpm.ps);
4022                                 return -ENOMEM;
4023                         }
4024                         rdev->pm.dpm.ps[i].ps_priv = ps;
4025                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4026                                                          non_clock_info,
4027                                                          power_info->pplib.ucNonClockSize);
4028                         idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4029                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4030                                 clock_info = (union pplib_clock_info *)
4031                                         (mode_info->atom_context->bios + data_offset +
4032                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4033                                          (idx[j] * power_info->pplib.ucClockInfoSize));
4034                                 ni_parse_pplib_clock_info(rdev,
4035                                                           &rdev->pm.dpm.ps[i], j,
4036                                                           clock_info);
4037                         }
4038                 }
4039         }
4040         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4041         return 0;
4042 }
4043
4044 int ni_dpm_init(struct radeon_device *rdev)
4045 {
4046         struct rv7xx_power_info *pi;
4047         struct evergreen_power_info *eg_pi;
4048         struct ni_power_info *ni_pi;
4049         struct atom_clock_dividers dividers;
4050         int ret;
4051
4052         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4053         if (ni_pi == NULL)
4054                 return -ENOMEM;
4055         rdev->pm.dpm.priv = ni_pi;
4056         eg_pi = &ni_pi->eg;
4057         pi = &eg_pi->rv7xx;
4058
4059         rv770_get_max_vddc(rdev);
4060
4061         eg_pi->ulv.supported = false;
4062         pi->acpi_vddc = 0;
4063         eg_pi->acpi_vddci = 0;
4064         pi->min_vddc_in_table = 0;
4065         pi->max_vddc_in_table = 0;
4066
4067         ret = r600_get_platform_caps(rdev);
4068         if (ret)
4069                 return ret;
4070
4071         ret = ni_parse_power_table(rdev);
4072         if (ret)
4073                 return ret;
4074         ret = r600_parse_extended_power_table(rdev);
4075         if (ret)
4076                 return ret;
4077
4078         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4079                 kcalloc(4,
4080                         sizeof(struct radeon_clock_voltage_dependency_entry),
4081                         GFP_KERNEL);
4082         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4083                 r600_free_extended_power_table(rdev);
4084                 return -ENOMEM;
4085         }
4086         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4087         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4088         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4089         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4090         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4091         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4092         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4093         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4094         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4095
4096         ni_patch_dependency_tables_based_on_leakage(rdev);
4097
4098         if (rdev->pm.dpm.voltage_response_time == 0)
4099                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4100         if (rdev->pm.dpm.backbias_response_time == 0)
4101                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4102
4103         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4104                                              0, false, &dividers);
4105         if (ret)
4106                 pi->ref_div = dividers.ref_div + 1;
4107         else
4108                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4109
4110         pi->rlp = RV770_RLP_DFLT;
4111         pi->rmp = RV770_RMP_DFLT;
4112         pi->lhp = RV770_LHP_DFLT;
4113         pi->lmp = RV770_LMP_DFLT;
4114
4115         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4116         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4117         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4118         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4119
4120         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4121         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4122         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4123         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4124
4125         eg_pi->smu_uvd_hs = true;
4126
4127         if (rdev->pdev->device == 0x6707) {
4128                 pi->mclk_strobe_mode_threshold = 55000;
4129                 pi->mclk_edc_enable_threshold = 55000;
4130                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4131         } else {
4132                 pi->mclk_strobe_mode_threshold = 40000;
4133                 pi->mclk_edc_enable_threshold = 40000;
4134                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4135         }
4136         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4137
4138         pi->voltage_control =
4139                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4140
4141         pi->mvdd_control =
4142                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4143
4144         eg_pi->vddci_control =
4145                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4146
4147         rv770_get_engine_memory_ss(rdev);
4148
4149         pi->asi = RV770_ASI_DFLT;
4150         pi->pasi = CYPRESS_HASI_DFLT;
4151         pi->vrc = CYPRESS_VRC_DFLT;
4152
4153         pi->power_gating = false;
4154
4155         pi->gfx_clock_gating = true;
4156
4157         pi->mg_clock_gating = true;
4158         pi->mgcgtssm = true;
4159         eg_pi->ls_clock_gating = false;
4160         eg_pi->sclk_deep_sleep = false;
4161
4162         pi->dynamic_pcie_gen2 = true;
4163
4164         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4165                 pi->thermal_protection = true;
4166         else
4167                 pi->thermal_protection = false;
4168
4169         pi->display_gap = true;
4170
4171         pi->dcodt = true;
4172
4173         pi->ulps = true;
4174
4175         eg_pi->dynamic_ac_timing = true;
4176         eg_pi->abm = true;
4177         eg_pi->mcls = true;
4178         eg_pi->light_sleep = true;
4179         eg_pi->memory_transition = true;
4180 #if defined(CONFIG_ACPI)
4181         eg_pi->pcie_performance_request =
4182                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4183 #else
4184         eg_pi->pcie_performance_request = false;
4185 #endif
4186
4187         eg_pi->dll_default_on = false;
4188
4189         eg_pi->sclk_deep_sleep = false;
4190
4191         pi->mclk_stutter_mode_threshold = 0;
4192
4193         pi->sram_end = SMC_RAM_END;
4194
4195         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4196         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4197         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4198         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4199         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4200         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4201         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4202         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4203
4204         ni_pi->cac_data.leakage_coefficients.at = 516;
4205         ni_pi->cac_data.leakage_coefficients.bt = 18;
4206         ni_pi->cac_data.leakage_coefficients.av = 51;
4207         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4208
4209         switch (rdev->pdev->device) {
4210         case 0x6700:
4211         case 0x6701:
4212         case 0x6702:
4213         case 0x6703:
4214         case 0x6718:
4215                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4216                 break;
4217         case 0x6705:
4218         case 0x6719:
4219         case 0x671D:
4220         case 0x671C:
4221         default:
4222                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4223                 break;
4224         case 0x6704:
4225         case 0x6706:
4226         case 0x6707:
4227         case 0x6708:
4228         case 0x6709:
4229                 ni_pi->cac_weights = &cac_weights_cayman_le;
4230                 break;
4231         }
4232
4233         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4234                 ni_pi->enable_power_containment = true;
4235                 ni_pi->enable_cac = true;
4236                 ni_pi->enable_sq_ramping = true;
4237         } else {
4238                 ni_pi->enable_power_containment = false;
4239                 ni_pi->enable_cac = false;
4240                 ni_pi->enable_sq_ramping = false;
4241         }
4242
4243         ni_pi->driver_calculate_cac_leakage = false;
4244         ni_pi->cac_configuration_required = true;
4245
4246         if (ni_pi->cac_configuration_required) {
4247                 ni_pi->support_cac_long_term_average = true;
4248                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4249                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4250         } else {
4251                 ni_pi->support_cac_long_term_average = false;
4252                 ni_pi->lta_window_size = 0;
4253                 ni_pi->lts_truncate = 0;
4254         }
4255
4256         ni_pi->use_power_boost_limit = true;
4257
4258         /* make sure dc limits are valid */
4259         if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4260             (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4261                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4262                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4263
4264         return 0;
4265 }
4266
4267 void ni_dpm_fini(struct radeon_device *rdev)
4268 {
4269         int i;
4270
4271         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4272                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4273         }
4274         kfree(rdev->pm.dpm.ps);
4275         kfree(rdev->pm.dpm.priv);
4276         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4277         r600_free_extended_power_table(rdev);
4278 }
4279
4280 void ni_dpm_print_power_state(struct radeon_device *rdev,
4281                               struct radeon_ps *rps)
4282 {
4283         struct ni_ps *ps = ni_get_ps(rps);
4284         struct rv7xx_pl *pl;
4285         int i;
4286
4287         r600_dpm_print_class_info(rps->class, rps->class2);
4288         r600_dpm_print_cap_info(rps->caps);
4289         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4290         for (i = 0; i < ps->performance_level_count; i++) {
4291                 pl = &ps->performance_levels[i];
4292                 if (rdev->family >= CHIP_TAHITI)
4293                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4294                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4295                 else
4296                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4297                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4298         }
4299         r600_dpm_print_ps_status(rdev, rps);
4300 }
4301
4302 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4303                                                     struct seq_file *m)
4304 {
4305         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4306         struct radeon_ps *rps = &eg_pi->current_rps;
4307         struct ni_ps *ps = ni_get_ps(rps);
4308         struct rv7xx_pl *pl;
4309         u32 current_index =
4310                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4311                 CURRENT_STATE_INDEX_SHIFT;
4312
4313         if (current_index >= ps->performance_level_count) {
4314                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4315         } else {
4316                 pl = &ps->performance_levels[current_index];
4317                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4318                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4319                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4320         }
4321 }
4322
4323 u32 ni_dpm_get_current_sclk(struct radeon_device *rdev)
4324 {
4325         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4326         struct radeon_ps *rps = &eg_pi->current_rps;
4327         struct ni_ps *ps = ni_get_ps(rps);
4328         struct rv7xx_pl *pl;
4329         u32 current_index =
4330                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4331                 CURRENT_STATE_INDEX_SHIFT;
4332
4333         if (current_index >= ps->performance_level_count) {
4334                 return 0;
4335         } else {
4336                 pl = &ps->performance_levels[current_index];
4337                 return pl->sclk;
4338         }
4339 }
4340
4341 u32 ni_dpm_get_current_mclk(struct radeon_device *rdev)
4342 {
4343         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4344         struct radeon_ps *rps = &eg_pi->current_rps;
4345         struct ni_ps *ps = ni_get_ps(rps);
4346         struct rv7xx_pl *pl;
4347         u32 current_index =
4348                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4349                 CURRENT_STATE_INDEX_SHIFT;
4350
4351         if (current_index >= ps->performance_level_count) {
4352                 return 0;
4353         } else {
4354                 pl = &ps->performance_levels[current_index];
4355                 return pl->mclk;
4356         }
4357 }
4358
4359 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4360 {
4361         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4362         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4363
4364         if (low)
4365                 return requested_state->performance_levels[0].sclk;
4366         else
4367                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4368 }
4369
4370 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4371 {
4372         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4373         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4374
4375         if (low)
4376                 return requested_state->performance_levels[0].mclk;
4377         else
4378                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4379 }
4380