2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/firmware.h>
29 #include "amdgpu_ucode.h"
31 #include "bif/bif_4_1_d.h"
32 #include "bif/bif_4_1_sh_mask.h"
34 #include "gmc/gmc_7_1_d.h"
35 #include "gmc/gmc_7_1_sh_mask.h"
37 #include "oss/oss_2_0_d.h"
38 #include "oss/oss_2_0_sh_mask.h"
40 #include "dce/dce_8_0_d.h"
41 #include "dce/dce_8_0_sh_mask.h"
43 #include "amdgpu_atombios.h"
45 static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev);
46 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
47 static int gmc_v7_0_wait_for_idle(void *handle);
51 static const u32 golden_settings_iceland_a11[] =
53 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
54 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
55 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
56 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
59 static const u32 iceland_mgcg_cgcg_init[] =
61 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
64 static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
66 switch (adev->asic_type) {
68 amdgpu_program_register_sequence(adev,
69 iceland_mgcg_cgcg_init,
70 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
71 amdgpu_program_register_sequence(adev,
72 golden_settings_iceland_a11,
73 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
80 static void gmc_v7_0_mc_stop(struct amdgpu_device *adev)
84 gmc_v7_0_wait_for_idle((void *)adev);
86 blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
87 if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
88 /* Block CPU access */
89 WREG32(mmBIF_FB_EN, 0);
91 blackout = REG_SET_FIELD(blackout,
92 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
93 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
95 /* wait for the MC to settle */
99 static void gmc_v7_0_mc_resume(struct amdgpu_device *adev)
103 /* unblackout the MC */
104 tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
105 tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
106 WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
107 /* allow CPU access */
108 tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
109 tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
110 WREG32(mmBIF_FB_EN, tmp);
114 * gmc_v7_0_init_microcode - load ucode images from disk
116 * @adev: amdgpu_device pointer
118 * Use the firmware interface to load the ucode images into
119 * the driver (not loaded into hw).
120 * Returns 0 on success, error on failure.
122 static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)
124 const char *chip_name;
130 switch (adev->asic_type) {
132 chip_name = "bonaire";
135 chip_name = "hawaii";
147 if (adev->asic_type == CHIP_TOPAZ)
148 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
150 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
152 err = reject_firmware(&adev->mc.fw, fw_name, adev->dev);
155 err = amdgpu_ucode_validate(adev->mc.fw);
159 pr_err("cik_mc: Failed to load firmware \"%s\"\n", fw_name);
160 release_firmware(adev->mc.fw);
167 * gmc_v7_0_mc_load_microcode - load MC ucode into the hw
169 * @adev: amdgpu_device pointer
171 * Load the GDDR MC ucode into the hw (CIK).
172 * Returns 0 on success, error on failure.
174 static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
176 const struct mc_firmware_header_v1_0 *hdr;
177 const __le32 *fw_data = NULL;
178 const __le32 *io_mc_regs = NULL;
180 int i, ucode_size, regs_size;
185 hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
186 amdgpu_ucode_print_mc_hdr(&hdr->header);
188 adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
189 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
190 io_mc_regs = (const __le32 *)
191 (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
192 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
193 fw_data = (const __le32 *)
194 (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
196 running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
199 /* reset the engine and set to writable */
200 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
201 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
203 /* load mc io regs */
204 for (i = 0; i < regs_size; i++) {
205 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
206 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
208 /* load the MC ucode */
209 for (i = 0; i < ucode_size; i++)
210 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
212 /* put the engine back into the active state */
213 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
214 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
215 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
217 /* wait for training to complete */
218 for (i = 0; i < adev->usec_timeout; i++) {
219 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
220 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
224 for (i = 0; i < adev->usec_timeout; i++) {
225 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
226 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
235 static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
236 struct amdgpu_mc *mc)
238 u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
241 if (mc->mc_vram_size > 0xFFC0000000ULL) {
242 /* leave room for at least 1024M GTT */
243 dev_warn(adev->dev, "limiting VRAM\n");
244 mc->real_vram_size = 0xFFC0000000ULL;
245 mc->mc_vram_size = 0xFFC0000000ULL;
247 amdgpu_vram_location(adev, &adev->mc, base);
248 amdgpu_gart_location(adev, mc);
252 * gmc_v7_0_mc_program - program the GPU memory controller
254 * @adev: amdgpu_device pointer
256 * Set the location of vram, gart, and AGP in the GPU's
257 * physical address space (CIK).
259 static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
265 for (i = 0, j = 0; i < 32; i++, j += 0x6) {
266 WREG32((0xb05 + j), 0x00000000);
267 WREG32((0xb06 + j), 0x00000000);
268 WREG32((0xb07 + j), 0x00000000);
269 WREG32((0xb08 + j), 0x00000000);
270 WREG32((0xb09 + j), 0x00000000);
272 WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
274 if (gmc_v7_0_wait_for_idle((void *)adev)) {
275 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
277 if (adev->mode_info.num_crtc) {
278 /* Lockout access through VGA aperture*/
279 tmp = RREG32(mmVGA_HDP_CONTROL);
280 tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
281 WREG32(mmVGA_HDP_CONTROL, tmp);
283 /* disable VGA render */
284 tmp = RREG32(mmVGA_RENDER_CONTROL);
285 tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
286 WREG32(mmVGA_RENDER_CONTROL, tmp);
288 /* Update configuration */
289 WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
290 adev->mc.vram_start >> 12);
291 WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
292 adev->mc.vram_end >> 12);
293 WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
294 adev->vram_scratch.gpu_addr >> 12);
295 WREG32(mmMC_VM_AGP_BASE, 0);
296 WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
297 WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
298 if (gmc_v7_0_wait_for_idle((void *)adev)) {
299 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
302 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
304 tmp = RREG32(mmHDP_MISC_CNTL);
305 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
306 WREG32(mmHDP_MISC_CNTL, tmp);
308 tmp = RREG32(mmHDP_HOST_PATH_CNTL);
309 WREG32(mmHDP_HOST_PATH_CNTL, tmp);
313 * gmc_v7_0_mc_init - initialize the memory controller driver params
315 * @adev: amdgpu_device pointer
317 * Look up the amount of vram, vram width, and decide how to place
318 * vram and gart within the GPU's physical address space (CIK).
319 * Returns 0 for success.
321 static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
323 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
324 if (!adev->mc.vram_width) {
326 int chansize, numchan;
328 /* Get VRAM informations */
329 tmp = RREG32(mmMC_ARB_RAMCFG);
330 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
335 tmp = RREG32(mmMC_SHARED_CHMAP);
336 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
366 adev->mc.vram_width = numchan * chansize;
368 /* Could aper size report 0 ? */
369 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
370 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
371 /* size in MB on si */
372 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
373 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
376 if (adev->flags & AMD_IS_APU) {
377 adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
378 adev->mc.aper_size = adev->mc.real_vram_size;
382 /* In case the PCI BAR is larger than the actual amount of vram */
383 adev->mc.visible_vram_size = adev->mc.aper_size;
384 if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
385 adev->mc.visible_vram_size = adev->mc.real_vram_size;
387 /* set the gart size */
388 if (amdgpu_gart_size == -1) {
389 switch (adev->asic_type) {
390 case CHIP_TOPAZ: /* no MM engines */
392 adev->mc.gart_size = 256ULL << 20;
394 #ifdef CONFIG_DRM_AMDGPU_CIK
395 case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
396 case CHIP_HAWAII: /* UVD, VCE do not support GPUVM */
397 case CHIP_KAVERI: /* UVD, VCE do not support GPUVM */
398 case CHIP_KABINI: /* UVD, VCE do not support GPUVM */
399 case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
400 adev->mc.gart_size = 1024ULL << 20;
405 adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
408 gmc_v7_0_vram_gtt_location(adev, &adev->mc);
415 * VMID 0 is the physical GPU addresses as used by the kernel.
416 * VMIDs 1-15 are used for userspace clients and are handled
417 * by the amdgpu vm/hsa code.
421 * gmc_v7_0_gart_flush_gpu_tlb - gart tlb flush callback
423 * @adev: amdgpu_device pointer
424 * @vmid: vm instance to flush
426 * Flush the TLB for the requested page table (CIK).
428 static void gmc_v7_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
431 /* flush hdp cache */
432 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
434 /* bits 0-15 are the VM contexts0-15 */
435 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
439 * gmc_v7_0_gart_set_pte_pde - update the page tables using MMIO
441 * @adev: amdgpu_device pointer
442 * @cpu_pt_addr: cpu address of the page table
443 * @gpu_page_idx: entry in the page table to update
444 * @addr: dst addr to write into pte/pde
445 * @flags: access flags
447 * Update the page tables using the CPU.
449 static int gmc_v7_0_gart_set_pte_pde(struct amdgpu_device *adev,
451 uint32_t gpu_page_idx,
455 void __iomem *ptr = (void *)cpu_pt_addr;
458 value = addr & 0xFFFFFFFFFFFFF000ULL;
460 writeq(value, ptr + (gpu_page_idx * 8));
465 static uint64_t gmc_v7_0_get_vm_pte_flags(struct amdgpu_device *adev,
468 uint64_t pte_flag = 0;
470 if (flags & AMDGPU_VM_PAGE_READABLE)
471 pte_flag |= AMDGPU_PTE_READABLE;
472 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
473 pte_flag |= AMDGPU_PTE_WRITEABLE;
474 if (flags & AMDGPU_VM_PAGE_PRT)
475 pte_flag |= AMDGPU_PTE_PRT;
480 static uint64_t gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
482 BUG_ON(addr & 0xFFFFFF0000000FFFULL);
487 * gmc_v8_0_set_fault_enable_default - update VM fault handling
489 * @adev: amdgpu_device pointer
490 * @value: true redirects VM faults to the default page
492 static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,
497 tmp = RREG32(mmVM_CONTEXT1_CNTL);
498 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
499 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
500 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
501 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
502 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
503 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
504 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
505 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
506 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
507 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
508 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
509 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
510 WREG32(mmVM_CONTEXT1_CNTL, tmp);
514 * gmc_v7_0_set_prt - set PRT VM fault
516 * @adev: amdgpu_device pointer
517 * @enable: enable/disable VM fault handling for PRT
519 static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)
523 if (enable && !adev->mc.prt_warning) {
524 dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
525 adev->mc.prt_warning = true;
528 tmp = RREG32(mmVM_PRT_CNTL);
529 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
530 CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
531 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
532 CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
533 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
534 TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
535 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
536 TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
537 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
538 L2_CACHE_STORE_INVALID_ENTRIES, enable);
539 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
540 L1_TLB_STORE_INVALID_ENTRIES, enable);
541 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
542 MASK_PDE0_FAULT, enable);
543 WREG32(mmVM_PRT_CNTL, tmp);
546 uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
547 uint32_t high = adev->vm_manager.max_pfn;
549 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
550 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
551 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
552 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
553 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
554 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
555 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
556 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
558 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
559 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
560 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
561 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
562 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
563 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
564 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
565 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
570 * gmc_v7_0_gart_enable - gart enable
572 * @adev: amdgpu_device pointer
574 * This sets up the TLBs, programs the page tables for VMID0,
575 * sets up the hw for VMIDs 1-15 which are allocated on
576 * demand, and sets up the global locations for the LDS, GDS,
577 * and GPUVM for FSA64 clients (CIK).
578 * Returns 0 for success, errors for failure.
580 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
585 if (adev->gart.robj == NULL) {
586 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
589 r = amdgpu_gart_table_vram_pin(adev);
592 /* Setup TLB control */
593 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
594 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
595 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
596 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
597 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
598 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
599 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
601 tmp = RREG32(mmVM_L2_CNTL);
602 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
603 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
604 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
605 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
606 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
607 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
608 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
609 WREG32(mmVM_L2_CNTL, tmp);
610 tmp = REG_SET_FIELD(0, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
611 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
612 WREG32(mmVM_L2_CNTL2, tmp);
614 field = adev->vm_manager.fragment_size;
615 tmp = RREG32(mmVM_L2_CNTL3);
616 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
617 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);
618 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);
619 WREG32(mmVM_L2_CNTL3, tmp);
621 WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
622 WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
623 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
624 WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
625 (u32)(adev->dummy_page.addr >> 12));
626 WREG32(mmVM_CONTEXT0_CNTL2, 0);
627 tmp = RREG32(mmVM_CONTEXT0_CNTL);
628 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
629 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
630 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
631 WREG32(mmVM_CONTEXT0_CNTL, tmp);
637 /* empty context1-15 */
638 /* FIXME start with 4G, once using 2 level pt switch to full
641 /* set vm size, must be a multiple of 4 */
642 WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
643 WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
644 for (i = 1; i < 16; i++) {
646 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
647 adev->gart.table_addr >> 12);
649 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
650 adev->gart.table_addr >> 12);
653 /* enable context1-15 */
654 WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
655 (u32)(adev->dummy_page.addr >> 12));
656 WREG32(mmVM_CONTEXT1_CNTL2, 4);
657 tmp = RREG32(mmVM_CONTEXT1_CNTL);
658 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
659 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
660 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
661 adev->vm_manager.block_size - 9);
662 WREG32(mmVM_CONTEXT1_CNTL, tmp);
663 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
664 gmc_v7_0_set_fault_enable_default(adev, false);
666 gmc_v7_0_set_fault_enable_default(adev, true);
668 if (adev->asic_type == CHIP_KAVERI) {
669 tmp = RREG32(mmCHUB_CONTROL);
671 WREG32(mmCHUB_CONTROL, tmp);
674 gmc_v7_0_gart_flush_gpu_tlb(adev, 0);
675 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
676 (unsigned)(adev->mc.gart_size >> 20),
677 (unsigned long long)adev->gart.table_addr);
678 adev->gart.ready = true;
682 static int gmc_v7_0_gart_init(struct amdgpu_device *adev)
686 if (adev->gart.robj) {
687 WARN(1, "R600 PCIE GART already initialized\n");
690 /* Initialize common gart structure */
691 r = amdgpu_gart_init(adev);
694 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
695 adev->gart.gart_pte_flags = 0;
696 return amdgpu_gart_table_vram_alloc(adev);
700 * gmc_v7_0_gart_disable - gart disable
702 * @adev: amdgpu_device pointer
704 * This disables all VM page table (CIK).
706 static void gmc_v7_0_gart_disable(struct amdgpu_device *adev)
710 /* Disable all tables */
711 WREG32(mmVM_CONTEXT0_CNTL, 0);
712 WREG32(mmVM_CONTEXT1_CNTL, 0);
713 /* Setup TLB control */
714 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
715 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
716 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
717 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
718 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
720 tmp = RREG32(mmVM_L2_CNTL);
721 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
722 WREG32(mmVM_L2_CNTL, tmp);
723 WREG32(mmVM_L2_CNTL2, 0);
724 amdgpu_gart_table_vram_unpin(adev);
728 * gmc_v7_0_gart_fini - vm fini callback
730 * @adev: amdgpu_device pointer
732 * Tears down the driver GART/VM setup (CIK).
734 static void gmc_v7_0_gart_fini(struct amdgpu_device *adev)
736 amdgpu_gart_table_vram_free(adev);
737 amdgpu_gart_fini(adev);
741 * gmc_v7_0_vm_decode_fault - print human readable fault info
743 * @adev: amdgpu_device pointer
744 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
745 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
747 * Print human readable fault information (CIK).
749 static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev,
750 u32 status, u32 addr, u32 mc_client)
753 u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
754 u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
756 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
757 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
759 mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
762 dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
763 protections, vmid, addr,
764 REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
766 "write" : "read", block, mc_client, mc_id);
770 static const u32 mc_cg_registers[] = {
771 mmMC_HUB_MISC_HUB_CG,
772 mmMC_HUB_MISC_SIP_CG,
776 mmMC_CITF_MISC_WR_CG,
777 mmMC_CITF_MISC_RD_CG,
778 mmMC_CITF_MISC_VM_CG,
782 static const u32 mc_cg_ls_en[] = {
783 MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
784 MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
785 MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
786 MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
787 ATC_MISC_CG__MEM_LS_ENABLE_MASK,
788 MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
789 MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
790 MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
791 VM_L2_CG__MEM_LS_ENABLE_MASK,
794 static const u32 mc_cg_en[] = {
795 MC_HUB_MISC_HUB_CG__ENABLE_MASK,
796 MC_HUB_MISC_SIP_CG__ENABLE_MASK,
797 MC_HUB_MISC_VM_CG__ENABLE_MASK,
798 MC_XPB_CLK_GAT__ENABLE_MASK,
799 ATC_MISC_CG__ENABLE_MASK,
800 MC_CITF_MISC_WR_CG__ENABLE_MASK,
801 MC_CITF_MISC_RD_CG__ENABLE_MASK,
802 MC_CITF_MISC_VM_CG__ENABLE_MASK,
803 VM_L2_CG__ENABLE_MASK,
806 static void gmc_v7_0_enable_mc_ls(struct amdgpu_device *adev,
812 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
813 orig = data = RREG32(mc_cg_registers[i]);
814 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
815 data |= mc_cg_ls_en[i];
817 data &= ~mc_cg_ls_en[i];
819 WREG32(mc_cg_registers[i], data);
823 static void gmc_v7_0_enable_mc_mgcg(struct amdgpu_device *adev,
829 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
830 orig = data = RREG32(mc_cg_registers[i]);
831 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
834 data &= ~mc_cg_en[i];
836 WREG32(mc_cg_registers[i], data);
840 static void gmc_v7_0_enable_bif_mgls(struct amdgpu_device *adev,
845 orig = data = RREG32_PCIE(ixPCIE_CNTL2);
847 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
848 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
849 data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
850 data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
851 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
853 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
854 data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
855 data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
856 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
860 WREG32_PCIE(ixPCIE_CNTL2, data);
863 static void gmc_v7_0_enable_hdp_mgcg(struct amdgpu_device *adev,
868 orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
870 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
871 data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
873 data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
876 WREG32(mmHDP_HOST_PATH_CNTL, data);
879 static void gmc_v7_0_enable_hdp_ls(struct amdgpu_device *adev,
884 orig = data = RREG32(mmHDP_MEM_POWER_LS);
886 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
887 data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
889 data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
892 WREG32(mmHDP_MEM_POWER_LS, data);
895 static int gmc_v7_0_convert_vram_type(int mc_seq_vram_type)
897 switch (mc_seq_vram_type) {
898 case MC_SEQ_MISC0__MT__GDDR1:
899 return AMDGPU_VRAM_TYPE_GDDR1;
900 case MC_SEQ_MISC0__MT__DDR2:
901 return AMDGPU_VRAM_TYPE_DDR2;
902 case MC_SEQ_MISC0__MT__GDDR3:
903 return AMDGPU_VRAM_TYPE_GDDR3;
904 case MC_SEQ_MISC0__MT__GDDR4:
905 return AMDGPU_VRAM_TYPE_GDDR4;
906 case MC_SEQ_MISC0__MT__GDDR5:
907 return AMDGPU_VRAM_TYPE_GDDR5;
908 case MC_SEQ_MISC0__MT__HBM:
909 return AMDGPU_VRAM_TYPE_HBM;
910 case MC_SEQ_MISC0__MT__DDR3:
911 return AMDGPU_VRAM_TYPE_DDR3;
913 return AMDGPU_VRAM_TYPE_UNKNOWN;
917 static int gmc_v7_0_early_init(void *handle)
919 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
921 gmc_v7_0_set_gart_funcs(adev);
922 gmc_v7_0_set_irq_funcs(adev);
924 adev->mc.shared_aperture_start = 0x2000000000000000ULL;
925 adev->mc.shared_aperture_end =
926 adev->mc.shared_aperture_start + (4ULL << 30) - 1;
927 adev->mc.private_aperture_start =
928 adev->mc.shared_aperture_end + 1;
929 adev->mc.private_aperture_end =
930 adev->mc.private_aperture_start + (4ULL << 30) - 1;
935 static int gmc_v7_0_late_init(void *handle)
937 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
939 if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
940 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
945 static int gmc_v7_0_sw_init(void *handle)
949 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
951 if (adev->flags & AMD_IS_APU) {
952 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
954 u32 tmp = RREG32(mmMC_SEQ_MISC0);
955 tmp &= MC_SEQ_MISC0__MT__MASK;
956 adev->mc.vram_type = gmc_v7_0_convert_vram_type(tmp);
959 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->mc.vm_fault);
963 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->mc.vm_fault);
967 /* Adjust VM size here.
968 * Currently set to 4GB ((1 << 20) 4k pages).
969 * Max GPUVM size for cayman and SI is 40 bits.
971 amdgpu_vm_adjust_size(adev, 64, 4);
972 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
974 /* Set the internal MC address mask
975 * This is the max address of the GPU's
976 * internal address space.
978 adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
980 adev->mc.stolen_size = 256 * 1024;
982 /* set DMA mask + need_dma32 flags.
983 * PCIE - can handle 40-bits.
984 * IGP - can handle 40-bits
985 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
987 adev->need_dma32 = false;
988 dma_bits = adev->need_dma32 ? 32 : 40;
989 r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
991 adev->need_dma32 = true;
993 pr_warn("amdgpu: No suitable DMA available\n");
995 r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
997 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
998 pr_warn("amdgpu: No coherent DMA available\n");
1001 r = gmc_v7_0_init_microcode(adev);
1003 DRM_ERROR("Failed to load mc firmware!\n");
1007 r = gmc_v7_0_mc_init(adev);
1011 /* Memory manager */
1012 r = amdgpu_bo_init(adev);
1016 r = gmc_v7_0_gart_init(adev);
1022 * VMID 0 is reserved for System
1023 * amdgpu graphics/compute will use VMIDs 1-7
1024 * amdkfd will use VMIDs 8-15
1026 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1027 adev->vm_manager.num_level = 1;
1028 amdgpu_vm_manager_init(adev);
1030 /* base offset of vram pages */
1031 if (adev->flags & AMD_IS_APU) {
1032 u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
1035 adev->vm_manager.vram_base_offset = tmp;
1037 adev->vm_manager.vram_base_offset = 0;
1043 static int gmc_v7_0_sw_fini(void *handle)
1045 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1047 amdgpu_vm_manager_fini(adev);
1048 gmc_v7_0_gart_fini(adev);
1049 amdgpu_gem_force_release(adev);
1050 amdgpu_bo_fini(adev);
1055 static int gmc_v7_0_hw_init(void *handle)
1058 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1060 gmc_v7_0_init_golden_registers(adev);
1062 gmc_v7_0_mc_program(adev);
1064 if (!(adev->flags & AMD_IS_APU)) {
1065 r = gmc_v7_0_mc_load_microcode(adev);
1067 DRM_ERROR("Failed to load MC firmware!\n");
1072 r = gmc_v7_0_gart_enable(adev);
1079 static int gmc_v7_0_hw_fini(void *handle)
1081 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1083 amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
1084 gmc_v7_0_gart_disable(adev);
1089 static int gmc_v7_0_suspend(void *handle)
1091 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1093 gmc_v7_0_hw_fini(adev);
1098 static int gmc_v7_0_resume(void *handle)
1101 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1103 r = gmc_v7_0_hw_init(adev);
1107 amdgpu_vm_reset_all_ids(adev);
1112 static bool gmc_v7_0_is_idle(void *handle)
1114 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1115 u32 tmp = RREG32(mmSRBM_STATUS);
1117 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1118 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
1124 static int gmc_v7_0_wait_for_idle(void *handle)
1128 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1130 for (i = 0; i < adev->usec_timeout; i++) {
1131 /* read MC_STATUS */
1132 tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1133 SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1134 SRBM_STATUS__MCC_BUSY_MASK |
1135 SRBM_STATUS__MCD_BUSY_MASK |
1136 SRBM_STATUS__VMC_BUSY_MASK);
1145 static int gmc_v7_0_soft_reset(void *handle)
1147 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1148 u32 srbm_soft_reset = 0;
1149 u32 tmp = RREG32(mmSRBM_STATUS);
1151 if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1152 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1153 SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1155 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1156 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
1157 if (!(adev->flags & AMD_IS_APU))
1158 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1159 SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1162 if (srbm_soft_reset) {
1163 gmc_v7_0_mc_stop(adev);
1164 if (gmc_v7_0_wait_for_idle((void *)adev)) {
1165 dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1169 tmp = RREG32(mmSRBM_SOFT_RESET);
1170 tmp |= srbm_soft_reset;
1171 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1172 WREG32(mmSRBM_SOFT_RESET, tmp);
1173 tmp = RREG32(mmSRBM_SOFT_RESET);
1177 tmp &= ~srbm_soft_reset;
1178 WREG32(mmSRBM_SOFT_RESET, tmp);
1179 tmp = RREG32(mmSRBM_SOFT_RESET);
1181 /* Wait a little for things to settle down */
1184 gmc_v7_0_mc_resume(adev);
1191 static int gmc_v7_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1192 struct amdgpu_irq_src *src,
1194 enum amdgpu_interrupt_state state)
1197 u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1198 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1199 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1200 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1201 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1202 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1205 case AMDGPU_IRQ_STATE_DISABLE:
1206 /* system context */
1207 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1209 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1211 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1213 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1215 case AMDGPU_IRQ_STATE_ENABLE:
1216 /* system context */
1217 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1219 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1221 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1223 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1232 static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
1233 struct amdgpu_irq_src *source,
1234 struct amdgpu_iv_entry *entry)
1236 u32 addr, status, mc_client;
1238 addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1239 status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1240 mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
1241 /* reset addr and status */
1242 WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1244 if (!addr && !status)
1247 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1248 gmc_v7_0_set_fault_enable_default(adev, false);
1250 if (printk_ratelimit()) {
1251 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1252 entry->src_id, entry->src_data[0]);
1253 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1255 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1257 gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client);
1263 static int gmc_v7_0_set_clockgating_state(void *handle,
1264 enum amd_clockgating_state state)
1267 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1269 if (state == AMD_CG_STATE_GATE)
1272 if (!(adev->flags & AMD_IS_APU)) {
1273 gmc_v7_0_enable_mc_mgcg(adev, gate);
1274 gmc_v7_0_enable_mc_ls(adev, gate);
1276 gmc_v7_0_enable_bif_mgls(adev, gate);
1277 gmc_v7_0_enable_hdp_mgcg(adev, gate);
1278 gmc_v7_0_enable_hdp_ls(adev, gate);
1283 static int gmc_v7_0_set_powergating_state(void *handle,
1284 enum amd_powergating_state state)
1289 static const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
1291 .early_init = gmc_v7_0_early_init,
1292 .late_init = gmc_v7_0_late_init,
1293 .sw_init = gmc_v7_0_sw_init,
1294 .sw_fini = gmc_v7_0_sw_fini,
1295 .hw_init = gmc_v7_0_hw_init,
1296 .hw_fini = gmc_v7_0_hw_fini,
1297 .suspend = gmc_v7_0_suspend,
1298 .resume = gmc_v7_0_resume,
1299 .is_idle = gmc_v7_0_is_idle,
1300 .wait_for_idle = gmc_v7_0_wait_for_idle,
1301 .soft_reset = gmc_v7_0_soft_reset,
1302 .set_clockgating_state = gmc_v7_0_set_clockgating_state,
1303 .set_powergating_state = gmc_v7_0_set_powergating_state,
1306 static const struct amdgpu_gart_funcs gmc_v7_0_gart_funcs = {
1307 .flush_gpu_tlb = gmc_v7_0_gart_flush_gpu_tlb,
1308 .set_pte_pde = gmc_v7_0_gart_set_pte_pde,
1309 .set_prt = gmc_v7_0_set_prt,
1310 .get_vm_pte_flags = gmc_v7_0_get_vm_pte_flags,
1311 .get_vm_pde = gmc_v7_0_get_vm_pde
1314 static const struct amdgpu_irq_src_funcs gmc_v7_0_irq_funcs = {
1315 .set = gmc_v7_0_vm_fault_interrupt_state,
1316 .process = gmc_v7_0_process_interrupt,
1319 static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev)
1321 if (adev->gart.gart_funcs == NULL)
1322 adev->gart.gart_funcs = &gmc_v7_0_gart_funcs;
1325 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)
1327 adev->mc.vm_fault.num_types = 1;
1328 adev->mc.vm_fault.funcs = &gmc_v7_0_irq_funcs;
1331 const struct amdgpu_ip_block_version gmc_v7_0_ip_block =
1333 .type = AMD_IP_BLOCK_TYPE_GMC,
1337 .funcs = &gmc_v7_0_ip_funcs,
1340 const struct amdgpu_ip_block_version gmc_v7_4_ip_block =
1342 .type = AMD_IP_BLOCK_TYPE_GMC,
1346 .funcs = &gmc_v7_0_ip_funcs,