drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <drm/drmP.h>
  26 #include "amdgpu.h"
  27 #include "amdgpu_vcn.h"
  28 #include "soc15d.h"
  29 #include "soc15_common.h"
  30
  31 #include "vega10/soc15ip.h"
  32 #include "raven1/VCN/vcn_1_0_offset.h"
  33 #include "raven1/VCN/vcn_1_0_sh_mask.h"
  34 #include "vega10/HDP/hdp_4_0_offset.h"
  35 #include "raven1/MMHUB/mmhub_9_1_offset.h"
  36 #include "raven1/MMHUB/mmhub_9_1_sh_mask.h"
  37
  38 static int vcn_v1_0_start(struct amdgpu_device *adev);
  39 static int vcn_v1_0_stop(struct amdgpu_device *adev);
  40 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  41 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  42 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev);
  43
  44 /**
  45  * vcn_v1_0_early_init - set function pointers
  46  *
  47  * @handle: amdgpu_device pointer
  48  *
  49  * Set ring and irq function pointers
  50  */
  51 static int vcn_v1_0_early_init(void *handle)
  52 {
  53         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  54
  55         adev->vcn.num_enc_rings = 2;
  56
  57         vcn_v1_0_set_dec_ring_funcs(adev);
  58         vcn_v1_0_set_enc_ring_funcs(adev);
  59         vcn_v1_0_set_irq_funcs(adev);
  60
  61         return 0;
  62 }
  63
  64 /**
  65  * vcn_v1_0_sw_init - sw init for VCN block
  66  *
  67  * @handle: amdgpu_device pointer
  68  *
  69  * Load firmware and sw initialization
  70  */
  71 static int vcn_v1_0_sw_init(void *handle)
  72 {
  73         struct amdgpu_ring *ring;
  74         int i, r;
  75         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  76
  77         /* VCN DEC TRAP */
  78         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VCN, 124, &adev->vcn.irq);
  79         if (r)
  80                 return r;
  81
  82         /* VCN ENC TRAP */
  83         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
  84                 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VCN, i + 119,
  85                                         &adev->vcn.irq);
  86                 if (r)
  87                         return r;
  88         }
  89
  90         r = amdgpu_vcn_sw_init(adev);
  91         if (r)
  92                 return r;
  93
  94         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
  95                 const struct common_firmware_header *hdr;
  96                 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
  97                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
  98                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
  99                 adev->firmware.fw_size +=
 100                         ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
 101                 DRM_INFO("PSP loading VCN firmware\n");
 102         }
 103
 104         r = amdgpu_vcn_resume(adev);
 105         if (r)
 106                 return r;
 107
 108         ring = &adev->vcn.ring_dec;
 109         sprintf(ring->name, "vcn_dec");
 110         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
 111         if (r)
 112                 return r;
 113
 114         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 115                 ring = &adev->vcn.ring_enc[i];
 116                 sprintf(ring->name, "vcn_enc%d", i);
 117                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
 118                 if (r)
 119                         return r;
 120         }
 121
 122         return r;
 123 }
 124
 125 /**
 126  * vcn_v1_0_sw_fini - sw fini for VCN block
 127  *
 128  * @handle: amdgpu_device pointer
 129  *
 130  * VCN suspend and free up sw allocation
 131  */
 132 static int vcn_v1_0_sw_fini(void *handle)
 133 {
 134         int r;
 135         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 136
 137         r = amdgpu_vcn_suspend(adev);
 138         if (r)
 139                 return r;
 140
 141         r = amdgpu_vcn_sw_fini(adev);
 142
 143         return r;
 144 }
 145
 146 /**
 147  * vcn_v1_0_hw_init - start and test VCN block
 148  *
 149  * @handle: amdgpu_device pointer
 150  *
 151  * Initialize the hardware, boot up the VCPU and do some testing
 152  */
 153 static int vcn_v1_0_hw_init(void *handle)
 154 {
 155         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 156         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 157         int i, r;
 158
 159         r = vcn_v1_0_start(adev);
 160         if (r)
 161                 goto done;
 162
 163         ring->ready = true;
 164         r = amdgpu_ring_test_ring(ring);
 165         if (r) {
 166                 ring->ready = false;
 167                 goto done;
 168         }
 169
 170         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 171                 ring = &adev->vcn.ring_enc[i];
 172                 ring->ready = true;
 173                 r = amdgpu_ring_test_ring(ring);
 174                 if (r) {
 175                         ring->ready = false;
 176                         goto done;
 177                 }
 178         }
 179
 180 done:
 181         if (!r)
 182                 DRM_INFO("VCN decode and encode initialized successfully.\n");
 183
 184         return r;
 185 }
 186
 187 /**
 188  * vcn_v1_0_hw_fini - stop the hardware block
 189  *
 190  * @handle: amdgpu_device pointer
 191  *
 192  * Stop the VCN block, mark ring as not ready any more
 193  */
 194 static int vcn_v1_0_hw_fini(void *handle)
 195 {
 196         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 197         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 198         int r;
 199
 200         r = vcn_v1_0_stop(adev);
 201         if (r)
 202                 return r;
 203
 204         ring->ready = false;
 205
 206         return 0;
 207 }
 208
 209 /**
 210  * vcn_v1_0_suspend - suspend VCN block
 211  *
 212  * @handle: amdgpu_device pointer
 213  *
 214  * HW fini and suspend VCN block
 215  */
 216 static int vcn_v1_0_suspend(void *handle)
 217 {
 218         int r;
 219         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 220
 221         r = vcn_v1_0_hw_fini(adev);
 222         if (r)
 223                 return r;
 224
 225         r = amdgpu_vcn_suspend(adev);
 226
 227         return r;
 228 }
 229
 230 /**
 231  * vcn_v1_0_resume - resume VCN block
 232  *
 233  * @handle: amdgpu_device pointer
 234  *
 235  * Resume firmware and hw init VCN block
 236  */
 237 static int vcn_v1_0_resume(void *handle)
 238 {
 239         int r;
 240         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 241
 242         r = amdgpu_vcn_resume(adev);
 243         if (r)
 244                 return r;
 245
 246         r = vcn_v1_0_hw_init(adev);
 247
 248         return r;
 249 }
 250
 251 /**
 252  * vcn_v1_0_mc_resume - memory controller programming
 253  *
 254  * @adev: amdgpu_device pointer
 255  *
 256  * Let the VCN memory controller know it's offsets
 257  */
 258 static void vcn_v1_0_mc_resume(struct amdgpu_device *adev)
 259 {
 260         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
 261         uint32_t offset;
 262
 263         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 264                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 265                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
 266                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 267                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
 268                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
 269                 offset = 0;
 270         } else {
 271                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 272                         lower_32_bits(adev->vcn.gpu_addr));
 273                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 274                         upper_32_bits(adev->vcn.gpu_addr));
 275                 offset = size;
 276                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 277                              AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 278         }
 279
 280         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
 281
 282         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 283                      lower_32_bits(adev->vcn.gpu_addr + offset));
 284         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 285                      upper_32_bits(adev->vcn.gpu_addr + offset));
 286         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
 287         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_HEAP_SIZE);
 288
 289         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 290                      lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE));
 291         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 292                      upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE));
 293         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
 294         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2,
 295                         AMDGPU_VCN_STACK_SIZE + (AMDGPU_VCN_SESSION_SIZE * 40));
 296
 297         WREG32_SOC15(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
 298                         adev->gfx.config.gb_addr_config);
 299         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
 300                         adev->gfx.config.gb_addr_config);
 301         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
 302                         adev->gfx.config.gb_addr_config);
 303 }
 304
 305 /**
 306  * vcn_v1_0_disable_clock_gating - disable VCN clock gating
 307  *
 308  * @adev: amdgpu_device pointer
 309  * @sw: enable SW clock gating
 310  *
 311  * Disable clock gating for VCN block
 312  */
 313 static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev, bool sw)
 314 {
 315         uint32_t data;
 316
 317         /* JPEG disable CGC */
 318         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 319
 320         if (sw)
 321                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 322         else
 323                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 324
 325         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 326         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 327         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
 328
 329         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 330         data &= ~(JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
 331         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
 332
 333         /* UVD disable CGC */
 334         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 335         if (sw)
 336                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 337         else
 338                 data &= ~ UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 339
 340         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 341         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 342         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 343
 344         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
 345         data &= ~(UVD_CGC_GATE__SYS_MASK
 346                 | UVD_CGC_GATE__UDEC_MASK
 347                 | UVD_CGC_GATE__MPEG2_MASK
 348                 | UVD_CGC_GATE__REGS_MASK
 349                 | UVD_CGC_GATE__RBC_MASK
 350                 | UVD_CGC_GATE__LMI_MC_MASK
 351                 | UVD_CGC_GATE__LMI_UMC_MASK
 352                 | UVD_CGC_GATE__IDCT_MASK
 353                 | UVD_CGC_GATE__MPRD_MASK
 354                 | UVD_CGC_GATE__MPC_MASK
 355                 | UVD_CGC_GATE__LBSI_MASK
 356                 | UVD_CGC_GATE__LRBBM_MASK
 357                 | UVD_CGC_GATE__UDEC_RE_MASK
 358                 | UVD_CGC_GATE__UDEC_CM_MASK
 359                 | UVD_CGC_GATE__UDEC_IT_MASK
 360                 | UVD_CGC_GATE__UDEC_DB_MASK
 361                 | UVD_CGC_GATE__UDEC_MP_MASK
 362                 | UVD_CGC_GATE__WCB_MASK
 363                 | UVD_CGC_GATE__VCPU_MASK
 364                 | UVD_CGC_GATE__SCPU_MASK);
 365         WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
 366
 367         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 368         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 369                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 370                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 371                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 372                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 373                 | UVD_CGC_CTRL__SYS_MODE_MASK
 374                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 375                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 376                 | UVD_CGC_CTRL__REGS_MODE_MASK
 377                 | UVD_CGC_CTRL__RBC_MODE_MASK
 378                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 379                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 380                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 381                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 382                 | UVD_CGC_CTRL__MPC_MODE_MASK
 383                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 384                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 385                 | UVD_CGC_CTRL__WCB_MODE_MASK
 386                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 387                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 388         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 389
 390         /* turn on */
 391         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
 392         data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 393                 | UVD_SUVD_CGC_GATE__SIT_MASK
 394                 | UVD_SUVD_CGC_GATE__SMP_MASK
 395                 | UVD_SUVD_CGC_GATE__SCM_MASK
 396                 | UVD_SUVD_CGC_GATE__SDB_MASK
 397                 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
 398                 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 399                 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
 400                 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 401                 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
 402                 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 403                 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
 404                 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 405                 | UVD_SUVD_CGC_GATE__SCLR_MASK
 406                 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
 407                 | UVD_SUVD_CGC_GATE__ENT_MASK
 408                 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 409                 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 410                 | UVD_SUVD_CGC_GATE__SITE_MASK
 411                 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 412                 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 413                 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 414                 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 415                 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
 416         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
 417
 418         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 419         data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 420                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 421                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 422                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 423                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 424                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 425                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 426                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 427                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 428                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 429         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 430 }
 431
 432 /**
 433  * vcn_v1_0_enable_clock_gating - enable VCN clock gating
 434  *
 435  * @adev: amdgpu_device pointer
 436  * @sw: enable SW clock gating
 437  *
 438  * Enable clock gating for VCN block
 439  */
 440 static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev, bool sw)
 441 {
 442         uint32_t data = 0;
 443
 444         /* enable JPEG CGC */
 445         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 446         if (sw)
 447                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 448         else
 449                 data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 450         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 451         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 452         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
 453
 454         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 455         data |= (JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
 456         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
 457
 458         /* enable UVD CGC */
 459         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 460         if (sw)
 461                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 462         else
 463                 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 464         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 465         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 466         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 467
 468         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 469         data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 470                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 471                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 472                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 473                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 474                 | UVD_CGC_CTRL__SYS_MODE_MASK
 475                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 476                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 477                 | UVD_CGC_CTRL__REGS_MODE_MASK
 478                 | UVD_CGC_CTRL__RBC_MODE_MASK
 479                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 480                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 481                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 482                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 483                 | UVD_CGC_CTRL__MPC_MODE_MASK
 484                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 485                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 486                 | UVD_CGC_CTRL__WCB_MODE_MASK
 487                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 488                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 489         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 490
 491         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 492         data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 493                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 494                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 495                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 496                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 497                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 498                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 499                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 500                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 501                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 502         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 503 }
 504
 505 /**
 506  * vcn_v1_0_start - start VCN block
 507  *
 508  * @adev: amdgpu_device pointer
 509  *
 510  * Setup and start the VCN block
 511  */
 512 static int vcn_v1_0_start(struct amdgpu_device *adev)
 513 {
 514         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 515         uint32_t rb_bufsz, tmp;
 516         uint32_t lmi_swap_cntl;
 517         int i, j, r;
 518
 519         /* disable byte swapping */
 520         lmi_swap_cntl = 0;
 521
 522         vcn_v1_0_mc_resume(adev);
 523
 524         /* disable clock gating */
 525         vcn_v1_0_disable_clock_gating(adev, true);
 526
 527         /* disable interupt */
 528         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
 529                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
 530
 531         /* stall UMC and register bus before resetting VCPU */
 532         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
 533                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
 534                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 535         mdelay(1);
 536
 537         /* put LMI, VCPU, RBC etc... into reset */
 538         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
 539                 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
 540                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
 541                 UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
 542                 UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
 543                 UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
 544                 UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
 545                 UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
 546                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
 547         mdelay(5);
 548
 549         /* initialize VCN memory controller */
 550         WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL,
 551                 (0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 552                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 553                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 554                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 555                 UVD_LMI_CTRL__REQ_MODE_MASK |
 556                 0x00100000L);
 557
 558 #ifdef __BIG_ENDIAN
 559         /* swap (8 in 32) RB and IB */
 560         lmi_swap_cntl = 0xa;
 561 #endif
 562         WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
 563
 564         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0, 0x40c2040);
 565         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA1, 0x0);
 566         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0, 0x40c2040);
 567         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB1, 0x0);
 568         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_ALU, 0);
 569         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX, 0x88);
 570
 571         /* take all subblocks out of reset, except VCPU */
 572         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
 573                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 574         mdelay(5);
 575
 576         /* enable VCPU clock */
 577         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL,
 578                         UVD_VCPU_CNTL__CLK_EN_MASK);
 579
 580         /* enable UMC */
 581         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
 582                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 583
 584         /* boot up the VCPU */
 585         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET, 0);
 586         mdelay(10);
 587
 588         for (i = 0; i < 10; ++i) {
 589                 uint32_t status;
 590
 591                 for (j = 0; j < 100; ++j) {
 592                         status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
 593                         if (status & 2)
 594                                 break;
 595                         mdelay(10);
 596                 }
 597                 r = 0;
 598                 if (status & 2)
 599                         break;
 600
 601                 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
 602                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
 603                                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
 604                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 605                 mdelay(10);
 606                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
 607                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 608                 mdelay(10);
 609                 r = -1;
 610         }
 611
 612         if (r) {
 613                 DRM_ERROR("VCN decode not responding, giving up!!!\n");
 614                 return r;
 615         }
 616         /* enable master interrupt */
 617         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
 618                 (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
 619                 ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
 620
 621         /* clear the bit 4 of VCN_STATUS */
 622         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
 623                         ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 624
 625         /* force RBC into idle state */
 626         rb_bufsz = order_base_2(ring->ring_size);
 627         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
 628         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
 629         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
 630         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
 631         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
 632         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
 633         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
 634
 635         /* set the write pointer delay */
 636         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
 637
 638         /* set the wb address */
 639         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
 640                         (upper_32_bits(ring->gpu_addr) >> 2));
 641
 642         /* programm the RB_BASE for ring buffer */
 643         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
 644                         lower_32_bits(ring->gpu_addr));
 645         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
 646                         upper_32_bits(ring->gpu_addr));
 647
 648         /* Initialize the ring buffer's read and write pointers */
 649         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
 650
 651         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
 652         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
 653                         lower_32_bits(ring->wptr));
 654
 655         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
 656                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
 657
 658         ring = &adev->vcn.ring_enc[0];
 659         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
 660         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
 661         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
 662         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 663         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
 664
 665         ring = &adev->vcn.ring_enc[1];
 666         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
 667         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
 668         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
 669         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
 670         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
 671
 672         return 0;
 673 }
 674
 675 /**
 676  * vcn_v1_0_stop - stop VCN block
 677  *
 678  * @adev: amdgpu_device pointer
 679  *
 680  * stop the VCN block
 681  */
 682 static int vcn_v1_0_stop(struct amdgpu_device *adev)
 683 {
 684         /* force RBC into idle state */
 685         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, 0x11010101);
 686
 687         /* Stall UMC and register bus before resetting VCPU */
 688         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
 689                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
 690                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 691         mdelay(1);
 692
 693         /* put VCPU into reset */
 694         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
 695                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 696         mdelay(5);
 697
 698         /* disable VCPU clock */
 699         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL, 0x0);
 700
 701         /* Unstall UMC and register bus */
 702         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
 703                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 704
 705         /* enable clock gating */
 706         vcn_v1_0_enable_clock_gating(adev, true);
 707
 708         return 0;
 709 }
 710
 711 static int vcn_v1_0_set_clockgating_state(void *handle,
 712                                           enum amd_clockgating_state state)
 713 {
 714         /* needed for driver unload*/
 715         return 0;
 716 }
 717
 718 /**
 719  * vcn_v1_0_dec_ring_get_rptr - get read pointer
 720  *
 721  * @ring: amdgpu_ring pointer
 722  *
 723  * Returns the current hardware read pointer
 724  */
 725 static uint64_t vcn_v1_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
 726 {
 727         struct amdgpu_device *adev = ring->adev;
 728
 729         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
 730 }
 731
 732 /**
 733  * vcn_v1_0_dec_ring_get_wptr - get write pointer
 734  *
 735  * @ring: amdgpu_ring pointer
 736  *
 737  * Returns the current hardware write pointer
 738  */
 739 static uint64_t vcn_v1_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
 740 {
 741         struct amdgpu_device *adev = ring->adev;
 742
 743         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
 744 }
 745
 746 /**
 747  * vcn_v1_0_dec_ring_set_wptr - set write pointer
 748  *
 749  * @ring: amdgpu_ring pointer
 750  *
 751  * Commits the write pointer to the hardware
 752  */
 753 static void vcn_v1_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
 754 {
 755         struct amdgpu_device *adev = ring->adev;
 756
 757         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
 758 }
 759
 760 /**
 761  * vcn_v1_0_dec_ring_insert_start - insert a start command
 762  *
 763  * @ring: amdgpu_ring pointer
 764  *
 765  * Write a start command to the ring.
 766  */
 767 static void vcn_v1_0_dec_ring_insert_start(struct amdgpu_ring *ring)
 768 {
 769         amdgpu_ring_write(ring,
 770                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 771         amdgpu_ring_write(ring, 0);
 772         amdgpu_ring_write(ring,
 773                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 774         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_START << 1);
 775 }
 776
 777 /**
 778  * vcn_v1_0_dec_ring_insert_end - insert a end command
 779  *
 780  * @ring: amdgpu_ring pointer
 781  *
 782  * Write a end command to the ring.
 783  */
 784 static void vcn_v1_0_dec_ring_insert_end(struct amdgpu_ring *ring)
 785 {
 786         amdgpu_ring_write(ring,
 787                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 788         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_END << 1);
 789 }
 790
 791 /**
 792  * vcn_v1_0_dec_ring_emit_fence - emit an fence & trap command
 793  *
 794  * @ring: amdgpu_ring pointer
 795  * @fence: fence to emit
 796  *
 797  * Write a fence and a trap command to the ring.
 798  */
 799 static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 800                                      unsigned flags)
 801 {
 802         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 803
 804         amdgpu_ring_write(ring,
 805                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
 806         amdgpu_ring_write(ring, seq);
 807         amdgpu_ring_write(ring,
 808                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 809         amdgpu_ring_write(ring, addr & 0xffffffff);
 810         amdgpu_ring_write(ring,
 811                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
 812         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
 813         amdgpu_ring_write(ring,
 814                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 815         amdgpu_ring_write(ring, VCN_DEC_CMD_FENCE << 1);
 816
 817         amdgpu_ring_write(ring,
 818                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 819         amdgpu_ring_write(ring, 0);
 820         amdgpu_ring_write(ring,
 821                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
 822         amdgpu_ring_write(ring, 0);
 823         amdgpu_ring_write(ring,
 824                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 825         amdgpu_ring_write(ring, VCN_DEC_CMD_TRAP << 1);
 826 }
 827
 828 /**
 829  * vcn_v1_0_dec_ring_hdp_invalidate - emit an hdp invalidate
 830  *
 831  * @ring: amdgpu_ring pointer
 832  *
 833  * Emits an hdp invalidate.
 834  */
 835 static void vcn_v1_0_dec_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 836 {
 837         amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0), 0));
 838         amdgpu_ring_write(ring, 1);
 839 }
 840
 841 /**
 842  * vcn_v1_0_dec_ring_emit_ib - execute indirect buffer
 843  *
 844  * @ring: amdgpu_ring pointer
 845  * @ib: indirect buffer to execute
 846  *
 847  * Write ring commands to execute the indirect buffer
 848  */
 849 static void vcn_v1_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
 850                                   struct amdgpu_ib *ib,
 851                                   unsigned vm_id, bool ctx_switch)
 852 {
 853         amdgpu_ring_write(ring,
 854                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
 855         amdgpu_ring_write(ring, vm_id);
 856
 857         amdgpu_ring_write(ring,
 858                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
 859         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 860         amdgpu_ring_write(ring,
 861                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));
 862         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 863         amdgpu_ring_write(ring,
 864                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_IB_SIZE), 0));
 865         amdgpu_ring_write(ring, ib->length_dw);
 866 }
 867
 868 static void vcn_v1_0_dec_vm_reg_write(struct amdgpu_ring *ring,
 869                                 uint32_t data0, uint32_t data1)
 870 {
 871         amdgpu_ring_write(ring,
 872                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 873         amdgpu_ring_write(ring, data0);
 874         amdgpu_ring_write(ring,
 875                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
 876         amdgpu_ring_write(ring, data1);
 877         amdgpu_ring_write(ring,
 878                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 879         amdgpu_ring_write(ring, VCN_DEC_CMD_WRITE_REG << 1);
 880 }
 881
 882 static void vcn_v1_0_dec_vm_reg_wait(struct amdgpu_ring *ring,
 883                                 uint32_t data0, uint32_t data1, uint32_t mask)
 884 {
 885         amdgpu_ring_write(ring,
 886                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 887         amdgpu_ring_write(ring, data0);
 888         amdgpu_ring_write(ring,
 889                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
 890         amdgpu_ring_write(ring, data1);
 891         amdgpu_ring_write(ring,
 892                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH8), 0));
 893         amdgpu_ring_write(ring, mask);
 894         amdgpu_ring_write(ring,
 895                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 896         amdgpu_ring_write(ring, VCN_DEC_CMD_REG_READ_COND_WAIT << 1);
 897 }
 898
 899 static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 900                                         unsigned vm_id, uint64_t pd_addr)
 901 {
 902         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
 903         uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
 904         uint32_t data0, data1, mask;
 905         unsigned eng = ring->vm_inv_eng;
 906
 907         pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
 908         pd_addr |= AMDGPU_PTE_VALID;
 909
 910         data0 = (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2;
 911         data1 = upper_32_bits(pd_addr);
 912         vcn_v1_0_dec_vm_reg_write(ring, data0, data1);
 913
 914         data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
 915         data1 = lower_32_bits(pd_addr);
 916         vcn_v1_0_dec_vm_reg_write(ring, data0, data1);
 917
 918         data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
 919         data1 = lower_32_bits(pd_addr);
 920         mask = 0xffffffff;
 921         vcn_v1_0_dec_vm_reg_wait(ring, data0, data1, mask);
 922
 923         /* flush TLB */
 924         data0 = (hub->vm_inv_eng0_req + eng) << 2;
 925         data1 = req;
 926         vcn_v1_0_dec_vm_reg_write(ring, data0, data1);
 927
 928         /* wait for flush */
 929         data0 = (hub->vm_inv_eng0_ack + eng) << 2;
 930         data1 = 1 << vm_id;
 931         mask =  1 << vm_id;
 932         vcn_v1_0_dec_vm_reg_wait(ring, data0, data1, mask);
 933 }
 934
 935 /**
 936  * vcn_v1_0_enc_ring_get_rptr - get enc read pointer
 937  *
 938  * @ring: amdgpu_ring pointer
 939  *
 940  * Returns the current hardware enc read pointer
 941  */
 942 static uint64_t vcn_v1_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
 943 {
 944         struct amdgpu_device *adev = ring->adev;
 945
 946         if (ring == &adev->vcn.ring_enc[0])
 947                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
 948         else
 949                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
 950 }
 951
 952  /**
 953  * vcn_v1_0_enc_ring_get_wptr - get enc write pointer
 954  *
 955  * @ring: amdgpu_ring pointer
 956  *
 957  * Returns the current hardware enc write pointer
 958  */
 959 static uint64_t vcn_v1_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
 960 {
 961         struct amdgpu_device *adev = ring->adev;
 962
 963         if (ring == &adev->vcn.ring_enc[0])
 964                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
 965         else
 966                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
 967 }
 968
 969  /**
 970  * vcn_v1_0_enc_ring_set_wptr - set enc write pointer
 971  *
 972  * @ring: amdgpu_ring pointer
 973  *
 974  * Commits the enc write pointer to the hardware
 975  */
 976 static void vcn_v1_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
 977 {
 978         struct amdgpu_device *adev = ring->adev;
 979
 980         if (ring == &adev->vcn.ring_enc[0])
 981                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR,
 982                         lower_32_bits(ring->wptr));
 983         else
 984                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2,
 985                         lower_32_bits(ring->wptr));
 986 }
 987
 988 /**
 989  * vcn_v1_0_enc_ring_emit_fence - emit an enc fence & trap command
 990  *
 991  * @ring: amdgpu_ring pointer
 992  * @fence: fence to emit
 993  *
 994  * Write enc a fence and a trap command to the ring.
 995  */
 996 static void vcn_v1_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 997                         u64 seq, unsigned flags)
 998 {
 999         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1000
1001         amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
1002         amdgpu_ring_write(ring, addr);
1003         amdgpu_ring_write(ring, upper_32_bits(addr));
1004         amdgpu_ring_write(ring, seq);
1005         amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
1006 }
1007
1008 static void vcn_v1_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1009 {
1010         amdgpu_ring_write(ring, VCN_ENC_CMD_END);
1011 }
1012
1013 /**
1014  * vcn_v1_0_enc_ring_emit_ib - enc execute indirect buffer
1015  *
1016  * @ring: amdgpu_ring pointer
1017  * @ib: indirect buffer to execute
1018  *
1019  * Write enc ring commands to execute the indirect buffer
1020  */
1021 static void vcn_v1_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1022                 struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
1023 {
1024         amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
1025         amdgpu_ring_write(ring, vm_id);
1026         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1027         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1028         amdgpu_ring_write(ring, ib->length_dw);
1029 }
1030
1031 static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1032                          unsigned int vm_id, uint64_t pd_addr)
1033 {
1034         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1035         uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
1036         unsigned eng = ring->vm_inv_eng;
1037
1038         pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
1039         pd_addr |= AMDGPU_PTE_VALID;
1040
1041         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1042         amdgpu_ring_write(ring,
1043                           (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2);
1044         amdgpu_ring_write(ring, upper_32_bits(pd_addr));
1045
1046         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1047         amdgpu_ring_write(ring,
1048                           (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
1049         amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1050
1051         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1052         amdgpu_ring_write(ring,
1053                           (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
1054         amdgpu_ring_write(ring, 0xffffffff);
1055         amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1056
1057         /* flush TLB */
1058         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1059         amdgpu_ring_write(ring, (hub->vm_inv_eng0_req + eng) << 2);
1060         amdgpu_ring_write(ring, req);
1061
1062         /* wait for flush */
1063         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1064         amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
1065         amdgpu_ring_write(ring, 1 << vm_id);
1066         amdgpu_ring_write(ring, 1 << vm_id);
1067 }
1068
1069 static int vcn_v1_0_set_interrupt_state(struct amdgpu_device *adev,
1070                                         struct amdgpu_irq_src *source,
1071                                         unsigned type,
1072                                         enum amdgpu_interrupt_state state)
1073 {
1074         return 0;
1075 }
1076
1077 static int vcn_v1_0_process_interrupt(struct amdgpu_device *adev,
1078                                       struct amdgpu_irq_src *source,
1079                                       struct amdgpu_iv_entry *entry)
1080 {
1081         DRM_DEBUG("IH: VCN TRAP\n");
1082
1083         switch (entry->src_id) {
1084         case 124:
1085                 amdgpu_fence_process(&adev->vcn.ring_dec);
1086                 break;
1087         case 119:
1088                 amdgpu_fence_process(&adev->vcn.ring_enc[0]);
1089                 break;
1090         case 120:
1091                 amdgpu_fence_process(&adev->vcn.ring_enc[1]);
1092                 break;
1093         default:
1094                 DRM_ERROR("Unhandled interrupt: %d %d\n",
1095                           entry->src_id, entry->src_data[0]);
1096                 break;
1097         }
1098
1099         return 0;
1100 }
1101
1102 static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
1103         .name = "vcn_v1_0",
1104         .early_init = vcn_v1_0_early_init,
1105         .late_init = NULL,
1106         .sw_init = vcn_v1_0_sw_init,
1107         .sw_fini = vcn_v1_0_sw_fini,
1108         .hw_init = vcn_v1_0_hw_init,
1109         .hw_fini = vcn_v1_0_hw_fini,
1110         .suspend = vcn_v1_0_suspend,
1111         .resume = vcn_v1_0_resume,
1112         .is_idle = NULL /* vcn_v1_0_is_idle */,
1113         .wait_for_idle = NULL /* vcn_v1_0_wait_for_idle */,
1114         .check_soft_reset = NULL /* vcn_v1_0_check_soft_reset */,
1115         .pre_soft_reset = NULL /* vcn_v1_0_pre_soft_reset */,
1116         .soft_reset = NULL /* vcn_v1_0_soft_reset */,
1117         .post_soft_reset = NULL /* vcn_v1_0_post_soft_reset */,
1118         .set_clockgating_state = vcn_v1_0_set_clockgating_state,
1119         .set_powergating_state = NULL /* vcn_v1_0_set_powergating_state */,
1120 };
1121
1122 static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
1123         .type = AMDGPU_RING_TYPE_VCN_DEC,
1124         .align_mask = 0xf,
1125         .nop = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0),
1126         .support_64bit_ptrs = false,
1127         .vmhub = AMDGPU_MMHUB,
1128         .get_rptr = vcn_v1_0_dec_ring_get_rptr,
1129         .get_wptr = vcn_v1_0_dec_ring_get_wptr,
1130         .set_wptr = vcn_v1_0_dec_ring_set_wptr,
1131         .emit_frame_size =
1132                 2 + /* vcn_v1_0_dec_ring_emit_hdp_invalidate */
1133                 34 + /* vcn_v1_0_dec_ring_emit_vm_flush */
1134                 14 + 14 + /* vcn_v1_0_dec_ring_emit_fence x2 vm fence */
1135                 6,
1136         .emit_ib_size = 8, /* vcn_v1_0_dec_ring_emit_ib */
1137         .emit_ib = vcn_v1_0_dec_ring_emit_ib,
1138         .emit_fence = vcn_v1_0_dec_ring_emit_fence,
1139         .emit_vm_flush = vcn_v1_0_dec_ring_emit_vm_flush,
1140         .emit_hdp_invalidate = vcn_v1_0_dec_ring_emit_hdp_invalidate,
1141         .test_ring = amdgpu_vcn_dec_ring_test_ring,
1142         .test_ib = amdgpu_vcn_dec_ring_test_ib,
1143         .insert_nop = amdgpu_ring_insert_nop,
1144         .insert_start = vcn_v1_0_dec_ring_insert_start,
1145         .insert_end = vcn_v1_0_dec_ring_insert_end,
1146         .pad_ib = amdgpu_ring_generic_pad_ib,
1147         .begin_use = amdgpu_vcn_ring_begin_use,
1148         .end_use = amdgpu_vcn_ring_end_use,
1149 };
1150
1151 static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
1152         .type = AMDGPU_RING_TYPE_VCN_ENC,
1153         .align_mask = 0x3f,
1154         .nop = VCN_ENC_CMD_NO_OP,
1155         .support_64bit_ptrs = false,
1156         .vmhub = AMDGPU_MMHUB,
1157         .get_rptr = vcn_v1_0_enc_ring_get_rptr,
1158         .get_wptr = vcn_v1_0_enc_ring_get_wptr,
1159         .set_wptr = vcn_v1_0_enc_ring_set_wptr,
1160         .emit_frame_size =
1161                 17 + /* vcn_v1_0_enc_ring_emit_vm_flush */
1162                 5 + 5 + /* vcn_v1_0_enc_ring_emit_fence x2 vm fence */
1163                 1, /* vcn_v1_0_enc_ring_insert_end */
1164         .emit_ib_size = 5, /* vcn_v1_0_enc_ring_emit_ib */
1165         .emit_ib = vcn_v1_0_enc_ring_emit_ib,
1166         .emit_fence = vcn_v1_0_enc_ring_emit_fence,
1167         .emit_vm_flush = vcn_v1_0_enc_ring_emit_vm_flush,
1168         .test_ring = amdgpu_vcn_enc_ring_test_ring,
1169         .test_ib = amdgpu_vcn_enc_ring_test_ib,
1170         .insert_nop = amdgpu_ring_insert_nop,
1171         .insert_end = vcn_v1_0_enc_ring_insert_end,
1172         .pad_ib = amdgpu_ring_generic_pad_ib,
1173         .begin_use = amdgpu_vcn_ring_begin_use,
1174         .end_use = amdgpu_vcn_ring_end_use,
1175 };
1176
1177 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev)
1178 {
1179         adev->vcn.ring_dec.funcs = &vcn_v1_0_dec_ring_vm_funcs;
1180         DRM_INFO("VCN decode is enabled in VM mode\n");
1181 }
1182
1183 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev)
1184 {
1185         int i;
1186
1187         for (i = 0; i < adev->vcn.num_enc_rings; ++i)
1188                 adev->vcn.ring_enc[i].funcs = &vcn_v1_0_enc_ring_vm_funcs;
1189
1190         DRM_INFO("VCN encode is enabled in VM mode\n");
1191 }
1192
1193 static const struct amdgpu_irq_src_funcs vcn_v1_0_irq_funcs = {
1194         .set = vcn_v1_0_set_interrupt_state,
1195         .process = vcn_v1_0_process_interrupt,
1196 };
1197
1198 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev)
1199 {
1200         adev->vcn.irq.num_types = adev->vcn.num_enc_rings + 1;
1201         adev->vcn.irq.funcs = &vcn_v1_0_irq_funcs;
1202 }
1203
1204 const struct amdgpu_ip_block_version vcn_v1_0_ip_block =
1205 {
1206                 .type = AMD_IP_BLOCK_TYPE_VCN,
1207                 .major = 1,
1208                 .minor = 0,
1209                 .rev = 0,
1210                 .funcs = &vcn_v1_0_ip_funcs,
1211 };