1 /* SPDX-License-Identifier: GPL-2.0-or-later */
4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
6 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
7 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
8 * Adapted for Power Macintosh by Paul Mackerras.
9 * Low-level exception handlers and MMU support
10 * rewritten by Paul Mackerras.
11 * Copyright (C) 1996 Paul Mackerras.
12 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
14 * This file contains the low-level support and setup for the
15 * PowerPC platform, including trap and interrupt dispatch.
16 * (The PPC 8xx embedded CPUs use head_8xx.S instead.)
19 #include <linux/init.h>
20 #include <linux/pgtable.h>
24 #include <asm/cputable.h>
25 #include <asm/cache.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/ptrace.h>
31 #include <asm/kvm_book3s_asm.h>
32 #include <asm/export.h>
33 #include <asm/feature-fixups.h>
37 #define LOAD_BAT(n, reg, RA, RB) \
38 /* see the comment for clear_bats() -- Cort */ \
40 mtspr SPRN_IBAT##n##U,RA; \
41 mtspr SPRN_DBAT##n##U,RA; \
42 lwz RA,(n*16)+0(reg); \
43 lwz RB,(n*16)+4(reg); \
44 mtspr SPRN_IBAT##n##U,RA; \
45 mtspr SPRN_IBAT##n##L,RB; \
46 lwz RA,(n*16)+8(reg); \
47 lwz RB,(n*16)+12(reg); \
48 mtspr SPRN_DBAT##n##U,RA; \
49 mtspr SPRN_DBAT##n##L,RB
52 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
53 .stabs "head_book3s_32.S",N_SO,0,0,0f
58 * _start is defined this way because the XCOFF loader in the OpenFirmware
59 * on the powermac expects the entry point to be a procedure descriptor.
63 * These are here for legacy reasons, the kernel used to
64 * need to look like a coff function entry for the pmac
65 * but we're always started by some kind of bootloader now.
68 nop /* used by __secondary_hold on prep (mtx) and chrp smp */
69 nop /* used by __secondary_hold on prep (mtx) and chrp smp */
73 * Enter here with the kernel text, data and bss loaded starting at
74 * 0, running with virtual == physical mapping.
75 * r5 points to the prom entry point (the client interface handler
76 * address). Address translation is turned on, with the prom
77 * managing the hash table. Interrupts are disabled. The stack
78 * pointer (r1) points to just below the end of the half-meg region
79 * from 0x380000 - 0x400000, which is mapped in already.
81 * If we are booted from MacOS via BootX, we enter with the kernel
82 * image loaded somewhere, and the following values in registers:
83 * r3: 'BooX' (0x426f6f58)
84 * r4: virtual address of boot_infos_t
88 * This is jumped to on prep systems right after the kernel is relocated
89 * to its proper place in memory by the boot loader. The expected layout
91 * r3: ptr to residual data
92 * r4: initrd_start or if no initrd then 0
93 * r5: initrd_end - unused if r4 is 0
94 * r6: Start of command line string
95 * r7: End of command line string
97 * This just gets a minimal mmu environment setup so we can call
98 * start_here() to do the real work.
105 * We have to do any OF calls before we map ourselves to KERNELBASE,
106 * because OF may have I/O devices mapped into that area
107 * (particularly on CHRP).
112 #ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE
113 /* find out where we are now */
115 0: mflr r8 /* r8 = runtime addr here */
116 addis r8,r8,(_stext - 0b)@ha
117 addi r8,r8,(_stext - 0b)@l /* current runtime base addr */
119 #endif /* CONFIG_PPC_OF_BOOT_TRAMPOLINE */
121 /* We never return. We also hit that trap if trying to boot
122 * from OF while CONFIG_PPC_OF_BOOT_TRAMPOLINE isn't selected */
126 * Check for BootX signature when supporting PowerMac and branch to
127 * appropriate trampoline if it's present
129 #ifdef CONFIG_PPC_PMAC
136 #endif /* CONFIG_PPC_PMAC */
138 1: mr r31,r3 /* save device tree ptr */
142 * early_init() does the early machine identification and does
143 * the necessary low-level setup and clears the BSS
144 * -- Cort <cort@fsmlabs.com>
148 /* Switch MMU off, clear BATs and flush TLB. At this point, r3 contains
149 * the physical address we are running at, returned by early_init()
157 bl load_segment_registers
158 BEGIN_MMU_FTR_SECTION
161 END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
162 #if defined(CONFIG_BOOTX_TEXT)
165 #ifdef CONFIG_PPC_EARLY_DEBUG_CPM
168 #ifdef CONFIG_PPC_EARLY_DEBUG_USBGECKO
169 bl setup_usbgecko_bat
173 * Call setup_cpu for CPU 0 and initialize 6xx Idle
177 bl call_setup_cpu /* Call setup_cpu for this CPU */
183 * We need to run with _start at physical address 0.
184 * On CHRP, we are loaded at 0x10000 since OF on CHRP uses
185 * the exception vectors at 0 (and therefore this copy
186 * overwrites OF's exception vectors with our own).
187 * The MMU is off at this point.
191 addis r4,r3,KERNELBASE@h /* current address of _start */
192 lis r5,PHYSICAL_START@h
193 cmplw 0,r4,r5 /* already running at PHYSICAL_START? */
196 * we now have the 1st 16M of ram mapped with the bats.
197 * prep needs the mmu to be turned on here, but pmac already has it on.
198 * this shouldn't bother the pmac since it just gets turned on again
199 * as we jump to our code at KERNELBASE. -- Cort
200 * Actually no, pmac doesn't have it on any more. BootX enters with MMU
201 * off, and in other cases, we now turn it off before changing BATs above.
205 ori r0,r0,MSR_DR|MSR_IR|MSR_RI
208 ori r0,r0,start_here@l
210 RFI /* enables MMU */
213 * We need __secondary_hold as a place to hold the other cpus on
214 * an SMP machine, even when we are running a UP kernel.
216 . = 0xc0 /* for prep bootloader */
217 li r3,1 /* MTX only has 1 cpu */
218 .globl __secondary_hold
220 /* tell the master we're here */
221 stw r3,__secondary_hold_acknowledge@l(0)
224 /* wait until we're told to start */
227 /* our cpu # was at addr 0 - go */
228 mr r24,r3 /* cpu # */
232 #endif /* CONFIG_SMP */
234 .globl __secondary_hold_spinloop
235 __secondary_hold_spinloop:
237 .globl __secondary_hold_acknowledge
238 __secondary_hold_acknowledge:
242 /* core99 pmac starts the seconary here by changing the vector, and
243 putting it back to what it was (unknown_exception) when done. */
244 EXCEPTION(0x100, Reset, unknown_exception, EXC_XFER_STD)
248 * On CHRP, this is complicated by the fact that we could get a
249 * machine check inside RTAS, and we have no guarantee that certain
250 * critical registers will have the values we expect. The set of
251 * registers that might have bad values includes all the GPRs
252 * and all the BATs. We indicate that we are in RTAS by putting
253 * a non-zero value, the address of the exception frame to use,
254 * in thread.rtas_sp. The machine check handler checks thread.rtas_sp
255 * and uses its value if it is non-zero.
256 * (Other exception handlers assume that r1 is a valid kernel stack
257 * pointer when we take an exception from supervisor mode.)
264 #ifdef CONFIG_PPC_CHRP
265 #ifdef CONFIG_VMAP_STACK
267 mfspr r1, SPRN_SPRG_THREAD
273 mfspr r11, SPRN_SPRG_THREAD
274 lwz r11, RTAS_SP(r11)
278 #endif /* CONFIG_PPC_CHRP */
279 EXCEPTION_PROLOG_1 for_rtas=1
280 7: EXCEPTION_PROLOG_2
281 addi r3,r1,STACK_FRAME_OVERHEAD
282 #ifdef CONFIG_PPC_CHRP
283 beq cr1, machine_check_tramp
286 b machine_check_tramp
289 /* Data access exception. */
293 #ifdef CONFIG_VMAP_STACK
294 mtspr SPRN_SPRG_SCRATCH0,r10
295 mfspr r10, SPRN_SPRG_THREAD
296 BEGIN_MMU_FTR_SECTION
298 mfspr r10, SPRN_DSISR
300 #ifdef CONFIG_PPC_KUAP
301 andis. r10, r10, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH | DSISR_PROTFAULT)@h
303 andis. r10, r10, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h
305 mfspr r10, SPRN_SPRG_THREAD
307 .Lhash_page_dsi_cont:
310 END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
311 mtspr SPRN_SPRG_SCRATCH1,r11
314 mfspr r11, SPRN_DSISR
318 mfspr r11, SPRN_SRR1 /* check whether user or kernel */
321 andi. r11, r11, MSR_PR
324 b handle_page_fault_tramp_1
325 #else /* CONFIG_VMAP_STACK */
326 EXCEPTION_PROLOG handle_dar_dsisr=1
327 get_and_save_dar_dsisr_on_stack r4, r5, r11
328 BEGIN_MMU_FTR_SECTION
329 #ifdef CONFIG_PPC_KUAP
330 andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH | DSISR_PROTFAULT)@h
332 andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h
334 bne handle_page_fault_tramp_2 /* if not, try to put a PTE */
335 rlwinm r3, r5, 32 - 15, 21, 21 /* DSISR_STORE -> _PAGE_RW */
337 b handle_page_fault_tramp_1
339 b handle_page_fault_tramp_2
340 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
341 #endif /* CONFIG_VMAP_STACK */
343 /* Instruction access exception. */
347 #ifdef CONFIG_VMAP_STACK
348 mtspr SPRN_SPRG_SCRATCH0,r10
349 mtspr SPRN_SPRG_SCRATCH1,r11
350 mfspr r10, SPRN_SPRG_THREAD
353 mfspr r11, SPRN_SRR1 /* check whether user or kernel */
356 BEGIN_MMU_FTR_SECTION
357 andis. r11, r11, SRR1_ISI_NOPT@h /* no pte found? */
359 .Lhash_page_isi_cont:
360 mfspr r11, SPRN_SRR1 /* check whether user or kernel */
361 END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
362 andi. r11, r11, MSR_PR
366 #else /* CONFIG_VMAP_STACK */
368 andis. r0,r9,SRR1_ISI_NOPT@h /* no pte found? */
369 beq 1f /* if so, try to put a PTE */
370 li r3,0 /* into the hash table */
371 mr r4,r12 /* SRR0 is fault address */
372 BEGIN_MMU_FTR_SECTION
374 END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
375 #endif /* CONFIG_VMAP_STACK */
377 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
379 EXC_XFER_LITE(0x400, handle_page_fault)
381 /* External interrupt */
382 EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
384 /* Alignment exception */
388 EXCEPTION_PROLOG handle_dar_dsisr=1
389 save_dar_dsisr_on_stack r4, r5, r11
390 addi r3,r1,STACK_FRAME_OVERHEAD
391 b alignment_exception_tramp
393 /* Program check exception */
394 EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
396 /* Floating-point unavailable */
402 * Certain Freescale cores don't have a FPU and treat fp instructions
403 * as a FP Unavailable exception. Redirect to illegal/emulation handling.
406 END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
409 bl load_up_fpu /* if from user, just load it up */
410 b fast_exception_return
411 1: addi r3,r1,STACK_FRAME_OVERHEAD
412 EXC_XFER_LITE(0x800, kernel_fp_unavailable_exception)
415 EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
417 EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_STD)
418 EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_STD)
426 EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
427 EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_STD)
430 * The Altivec unavailable trap is at 0x0f20. Foo.
431 * We effectively remap it to 0x3000.
432 * We include an altivec unavailable exception vector even if
433 * not configured for Altivec, so that you can't panic a
434 * non-altivec kernel running on a machine with altivec just
435 * by executing an altivec instruction.
446 * Handle TLB miss for instruction on 603/603e.
447 * Note: we get an alternate set of r0 - r3 to use automatically.
453 * r1: linux style pte ( later becomes ppc hardware pte )
454 * r2: ptr to linux-style pte
457 /* Get PTE (linux-style) and check access */
459 #if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC)
460 lis r1, TASK_SIZE@h /* check if kernel address */
463 mfspr r2, SPRN_SPRG_PGDIR
464 li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC | _PAGE_USER
465 #if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC)
467 lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
468 li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
469 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
471 112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
472 lwz r2,0(r2) /* get pmd entry */
473 rlwinm. r2,r2,0,0,19 /* extract address of pte page */
474 beq- InstructionAddressInvalid /* return if no mapping */
475 rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */
476 lwz r0,0(r2) /* get linux-style pte */
477 andc. r1,r1,r0 /* check access & ~permission */
478 bne- InstructionAddressInvalid /* return if access not permitted */
479 /* Convert linux-style PTE to low word of PPC-style PTE */
480 rlwimi r0,r0,32-2,31,31 /* _PAGE_USER -> PP lsb */
481 ori r1, r1, 0xe06 /* clear out reserved bits */
482 andc r1, r0, r1 /* PP = user? 1 : 0 */
484 rlwinm r1,r1,0,~_PAGE_COHERENT /* clear M (coherence not required) */
485 END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
488 mfspr r3,SPRN_SRR1 /* Need to restore CR0 */
491 InstructionAddressInvalid:
493 rlwinm r1,r3,9,6,6 /* Get load/store bit */
496 mtspr SPRN_DSISR,r1 /* (shouldn't be needed) */
497 andi. r2,r3,0xFFFF /* Clear upper bits of SRR1 */
500 mfspr r1,SPRN_IMISS /* Get failing address */
501 rlwinm. r2,r2,0,31,31 /* Check for little endian access */
502 rlwimi r2,r2,1,30,30 /* change 1 -> 3 */
504 mtspr SPRN_DAR,r1 /* Set fault address */
505 mfmsr r0 /* Restore "normal" registers */
506 xoris r0,r0,MSR_TGPR>>16
507 mtcrf 0x80,r3 /* Restore CR0 */
512 * Handle TLB miss for DATA Load operation on 603/603e
518 * r1: linux style pte ( later becomes ppc hardware pte )
519 * r2: ptr to linux-style pte
522 /* Get PTE (linux-style) and check access */
524 lis r1, TASK_SIZE@h /* check if kernel address */
526 mfspr r2, SPRN_SPRG_PGDIR
527 li r1, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER
529 lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
530 li r1, _PAGE_PRESENT | _PAGE_ACCESSED
531 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
532 112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
533 lwz r2,0(r2) /* get pmd entry */
534 rlwinm. r2,r2,0,0,19 /* extract address of pte page */
535 beq- DataAddressInvalid /* return if no mapping */
536 rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */
537 lwz r0,0(r2) /* get linux-style pte */
538 andc. r1,r1,r0 /* check access & ~permission */
539 bne- DataAddressInvalid /* return if access not permitted */
541 * NOTE! We are assuming this is not an SMP system, otherwise
542 * we would need to update the pte atomically with lwarx/stwcx.
544 /* Convert linux-style PTE to low word of PPC-style PTE */
545 rlwinm r1,r0,32-9,30,30 /* _PAGE_RW -> PP msb */
546 rlwimi r0,r0,32-1,30,30 /* _PAGE_USER -> PP msb */
547 rlwimi r0,r0,32-1,31,31 /* _PAGE_USER -> PP lsb */
548 ori r1,r1,0xe04 /* clear out reserved bits */
549 andc r1,r0,r1 /* PP = user? rw? 1: 3: 0 */
551 rlwinm r1,r1,0,~_PAGE_COHERENT /* clear M (coherence not required) */
552 END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
554 mfspr r2,SPRN_SRR1 /* Need to restore CR0 */
556 BEGIN_MMU_FTR_SECTION
558 mfspr r1,SPRN_SPRG_603_LRU
559 rlwinm r2,r3,20,27,31 /* Get Address bits 15:19 */
563 mtspr SPRN_SPRG_603_LRU,r1
565 rlwimi r2,r0,31-14,14,14
567 END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
572 rlwinm r1,r3,9,6,6 /* Get load/store bit */
575 andi. r2,r3,0xFFFF /* Clear upper bits of SRR1 */
577 mfspr r1,SPRN_DMISS /* Get failing address */
578 rlwinm. r2,r2,0,31,31 /* Check for little endian access */
579 beq 20f /* Jump if big endian */
581 20: mtspr SPRN_DAR,r1 /* Set fault address */
582 mfmsr r0 /* Restore "normal" registers */
583 xoris r0,r0,MSR_TGPR>>16
584 mtcrf 0x80,r3 /* Restore CR0 */
589 * Handle TLB miss for DATA Store on 603/603e
595 * r1: linux style pte ( later becomes ppc hardware pte )
596 * r2: ptr to linux-style pte
599 /* Get PTE (linux-style) and check access */
601 lis r1, TASK_SIZE@h /* check if kernel address */
603 mfspr r2, SPRN_SPRG_PGDIR
604 li r1, _PAGE_RW | _PAGE_DIRTY | _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER
606 lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
607 li r1, _PAGE_RW | _PAGE_DIRTY | _PAGE_PRESENT | _PAGE_ACCESSED
608 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
609 112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
610 lwz r2,0(r2) /* get pmd entry */
611 rlwinm. r2,r2,0,0,19 /* extract address of pte page */
612 beq- DataAddressInvalid /* return if no mapping */
613 rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */
614 lwz r0,0(r2) /* get linux-style pte */
615 andc. r1,r1,r0 /* check access & ~permission */
616 bne- DataAddressInvalid /* return if access not permitted */
618 * NOTE! We are assuming this is not an SMP system, otherwise
619 * we would need to update the pte atomically with lwarx/stwcx.
621 /* Convert linux-style PTE to low word of PPC-style PTE */
622 rlwimi r0,r0,32-2,31,31 /* _PAGE_USER -> PP lsb */
623 li r1,0xe06 /* clear out reserved bits & PP msb */
624 andc r1,r0,r1 /* PP = user? 1: 0 */
626 rlwinm r1,r1,0,~_PAGE_COHERENT /* clear M (coherence not required) */
627 END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
629 mfspr r2,SPRN_SRR1 /* Need to restore CR0 */
631 BEGIN_MMU_FTR_SECTION
633 mfspr r1,SPRN_SPRG_603_LRU
634 rlwinm r2,r3,20,27,31 /* Get Address bits 15:19 */
638 mtspr SPRN_SPRG_603_LRU,r1
640 rlwimi r2,r0,31-14,14,14
642 END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
646 #ifndef CONFIG_ALTIVEC
647 #define altivec_assist_exception unknown_exception
650 #ifndef CONFIG_TAU_INT
651 #define TAUException unknown_exception
654 EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD)
655 EXCEPTION(0x1400, SMI, SMIException, EXC_XFER_STD)
656 EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
657 EXCEPTION(0x1600, Trap_16, altivec_assist_exception, EXC_XFER_STD)
658 EXCEPTION(0x1700, Trap_17, TAUException, EXC_XFER_STD)
659 EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)
660 EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)
661 EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_STD)
662 EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_STD)
663 EXCEPTION(0x1c00, Trap_1c, unknown_exception, EXC_XFER_STD)
664 EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
665 EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
666 EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
667 EXCEPTION(0x2000, RunMode, RunModeException, EXC_XFER_STD)
668 EXCEPTION(0x2100, Trap_21, unknown_exception, EXC_XFER_STD)
669 EXCEPTION(0x2200, Trap_22, unknown_exception, EXC_XFER_STD)
670 EXCEPTION(0x2300, Trap_23, unknown_exception, EXC_XFER_STD)
671 EXCEPTION(0x2400, Trap_24, unknown_exception, EXC_XFER_STD)
672 EXCEPTION(0x2500, Trap_25, unknown_exception, EXC_XFER_STD)
673 EXCEPTION(0x2600, Trap_26, unknown_exception, EXC_XFER_STD)
674 EXCEPTION(0x2700, Trap_27, unknown_exception, EXC_XFER_STD)
675 EXCEPTION(0x2800, Trap_28, unknown_exception, EXC_XFER_STD)
676 EXCEPTION(0x2900, Trap_29, unknown_exception, EXC_XFER_STD)
677 EXCEPTION(0x2a00, Trap_2a, unknown_exception, EXC_XFER_STD)
678 EXCEPTION(0x2b00, Trap_2b, unknown_exception, EXC_XFER_STD)
679 EXCEPTION(0x2c00, Trap_2c, unknown_exception, EXC_XFER_STD)
680 EXCEPTION(0x2d00, Trap_2d, unknown_exception, EXC_XFER_STD)
681 EXCEPTION(0x2e00, Trap_2e, unknown_exception, EXC_XFER_STD)
682 EXCEPTION(0x2f00, Trap_2f, unknown_exception, EXC_XFER_STD)
687 EXC_XFER_STD(0x200, machine_check_exception)
689 alignment_exception_tramp:
690 EXC_XFER_STD(0x600, alignment_exception)
692 handle_page_fault_tramp_1:
693 #ifdef CONFIG_VMAP_STACK
694 EXCEPTION_PROLOG_2 handle_dar_dsisr=1
699 handle_page_fault_tramp_2:
700 EXC_XFER_LITE(0x300, handle_page_fault)
702 #ifdef CONFIG_VMAP_STACK
703 .macro save_regs_thread thread
704 stw r0, THR0(\thread)
705 stw r3, THR3(\thread)
706 stw r4, THR4(\thread)
707 stw r5, THR5(\thread)
708 stw r6, THR6(\thread)
709 stw r8, THR8(\thread)
710 stw r9, THR9(\thread)
712 stw r0, THLR(\thread)
714 stw r0, THCTR(\thread)
717 .macro restore_regs_thread thread
718 lwz r0, THLR(\thread)
720 lwz r0, THCTR(\thread)
722 lwz r0, THR0(\thread)
723 lwz r3, THR3(\thread)
724 lwz r4, THR4(\thread)
725 lwz r5, THR5(\thread)
726 lwz r6, THR6(\thread)
727 lwz r8, THR8(\thread)
728 lwz r9, THR9(\thread)
737 rlwinm r3, r3, 32 - 15, _PAGE_RW /* DSISR_STORE -> _PAGE_RW */
739 mfspr r10, SPRN_SPRG_THREAD
740 restore_regs_thread r10
741 b .Lhash_page_dsi_cont
745 mfspr r10, SPRN_SPRG_THREAD
751 mfspr r10, SPRN_SPRG_THREAD
752 restore_regs_thread r10
754 b .Lhash_page_isi_cont
756 .globl fast_hash_page_return
757 fast_hash_page_return:
758 andis. r10, r9, SRR1_ISI_NOPT@h /* Set on ISI, cleared on DSI */
759 mfspr r10, SPRN_SPRG_THREAD
760 restore_regs_thread r10
766 mfspr r10, SPRN_SPRG_SCRATCH0
771 mfspr r11, SPRN_SPRG_SCRATCH1
772 mfspr r10, SPRN_SPRG_SCRATCH0
776 vmap_stack_overflow_exception
781 #ifdef CONFIG_ALTIVEC
783 bl load_up_altivec /* if from user, just load it up */
784 b fast_exception_return
785 #endif /* CONFIG_ALTIVEC */
786 1: addi r3,r1,STACK_FRAME_OVERHEAD
787 EXC_XFER_LITE(0xf20, altivec_unavailable_exception)
791 addi r3,r1,STACK_FRAME_OVERHEAD
792 EXC_XFER_STD(0xf00, performance_monitor_exception)
796 * This code is jumped to from the startup code to copy
797 * the kernel image to physical address PHYSICAL_START.
800 addis r9,r26,klimit@ha /* fetch klimit */
802 addis r25,r25,-KERNELBASE@h
803 lis r3,PHYSICAL_START@h /* Destination base address */
804 li r6,0 /* Destination offset */
805 li r5,0x4000 /* # bytes of memory to copy */
806 bl copy_and_flush /* copy the first 0x4000 bytes */
807 addi r0,r3,4f@l /* jump to the address of 4f */
808 mtctr r0 /* in copy and do the rest. */
809 bctr /* jump to the copy */
811 bl copy_and_flush /* copy the rest */
815 * Copy routine used to copy the kernel to start at physical address 0
816 * and flush and invalidate the caches as needed.
817 * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
818 * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
820 _ENTRY(copy_and_flush)
823 4: li r0,L1_CACHE_BYTES/4
825 3: addi r6,r6,4 /* copy a cache line */
829 dcbst r6,r3 /* write it to memory */
831 icbi r6,r3 /* flush the icache line */
834 sync /* additional sync needed on g4 */
841 .globl __secondary_start_mpc86xx
842 __secondary_start_mpc86xx:
844 stw r3, __secondary_hold_acknowledge@l(0)
845 mr r24, r3 /* cpu # */
848 .globl __secondary_start_pmac_0
849 __secondary_start_pmac_0:
850 /* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
859 /* on powersurge, we come in here with IR=0 and DR=1, and DBAT 0
860 set to map the 0xf0000000 - 0xffffffff region */
862 rlwinm r0,r0,0,28,26 /* clear DR (0x10) */
866 .globl __secondary_start
868 /* Copy some CPU settings from CPU 0 */
869 bl __restore_cpu_setup
873 bl call_setup_cpu /* Call setup_cpu for this CPU */
877 /* get current's stack and current */
878 lis r2,secondary_current@ha
880 lwz r2,secondary_current@l(r2)
882 lwz r1,TASK_STACK(r1)
885 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
890 /* load up the MMU */
891 bl load_segment_registers
894 /* ptr to phys current thread */
896 addi r4,r4,THREAD /* phys address of our thread_struct */
897 mtspr SPRN_SPRG_THREAD,r4
898 lis r4, (swapper_pg_dir - PAGE_OFFSET)@h
899 ori r4, r4, (swapper_pg_dir - PAGE_OFFSET)@l
900 mtspr SPRN_SPRG_PGDIR, r4
902 /* enable MMU and jump to start_secondary */
904 lis r3,start_secondary@h
905 ori r3,r3,start_secondary@l
909 #endif /* CONFIG_SMP */
911 #ifdef CONFIG_KVM_BOOK3S_HANDLER
912 #include "../kvm/book3s_rmhandlers.S"
916 * Load stuff into the MMU. Intended to be called with
920 sync /* Force all PTE updates to finish */
922 tlbia /* Clear all TLB entries */
923 sync /* wait for tlbia/tlbie to finish */
924 TLBSYNC /* ... on all CPUs */
925 /* Load the SDR1 register (hash table base & size) */
926 lis r6, early_hash - PAGE_OFFSET@h
927 ori r6, r6, 3 /* 256kB table */
930 addis r3, r3, Hash@ha
935 sync /* Force all PTE updates to finish */
937 tlbia /* Clear all TLB entries */
938 sync /* wait for tlbia/tlbie to finish */
939 TLBSYNC /* ... on all CPUs */
940 /* Load the SDR1 register (hash table base & size) */
946 /* Load the BAT registers with the values set up by MMU_init. */
954 BEGIN_MMU_FTR_SECTION
959 END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
962 _GLOBAL(load_segment_registers)
963 li r0, NUM_USER_SEGMENTS /* load up user segment register values */
964 mtctr r0 /* for context 0 */
965 li r3, 0 /* Kp = 0, Ks = 0, VSID = 0 */
966 #ifdef CONFIG_PPC_KUEP
967 oris r3, r3, SR_NX@h /* Set Nx */
969 #ifdef CONFIG_PPC_KUAP
970 oris r3, r3, SR_KS@h /* Set Ks */
974 addi r3, r3, 0x111 /* increment VSID */
975 addis r4, r4, 0x1000 /* address of next segment */
977 li r0, 16 - NUM_USER_SEGMENTS /* load up kernel segment registers */
978 mtctr r0 /* for context 0 */
979 rlwinm r3, r3, 0, ~SR_NX /* Nx = 0 */
980 rlwinm r3, r3, 0, ~SR_KS /* Ks = 0 */
981 oris r3, r3, SR_KP@h /* Kp = 1 */
983 addi r3, r3, 0x111 /* increment VSID */
984 addis r4, r4, 0x1000 /* address of next segment */
989 * This is where the main kernel code starts.
994 ori r2,r2,init_task@l
995 /* Set up for using our exception vectors */
996 /* ptr to phys current thread */
998 addi r4,r4,THREAD /* init task's THREAD */
999 mtspr SPRN_SPRG_THREAD,r4
1000 lis r4, (swapper_pg_dir - PAGE_OFFSET)@h
1001 ori r4, r4, (swapper_pg_dir - PAGE_OFFSET)@l
1002 mtspr SPRN_SPRG_PGDIR, r4
1005 lis r1,init_thread_union@ha
1006 addi r1,r1,init_thread_union@l
1008 stwu r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
1010 * Do early platform-specific initialization,
1011 * and set up the MMU.
1021 bl MMU_init_hw_patch
1024 * Go back to running unmapped so we can load up new values
1025 * for SDR1 (hash table pointer) and the segment registers
1026 * and change to using our exception vectors.
1031 li r3,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1037 /* Load up the kernel context */
1040 #ifdef CONFIG_BDI_SWITCH
1041 /* Add helper information for the Abatron bdiGDB debugger.
1042 * We do this here because we know the mmu is disabled, and
1043 * will be enabled for real in just a few instructions.
1045 lis r5, abatron_pteptrs@h
1046 ori r5, r5, abatron_pteptrs@l
1047 stw r5, 0xf0(0) /* This much match your Abatron config */
1048 lis r6, swapper_pg_dir@h
1049 ori r6, r6, swapper_pg_dir@l
1052 #endif /* CONFIG_BDI_SWITCH */
1054 /* Now turn on the MMU for real! */
1056 lis r3,start_kernel@h
1057 ori r3,r3,start_kernel@l
1063 * void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
1065 * Set up the segment registers for a new context.
1067 _ENTRY(switch_mmu_context)
1068 lwz r3,MMCONTEXTID(r4)
1071 mulli r3,r3,897 /* multiply context by skew factor */
1072 rlwinm r3,r3,4,8,27 /* VSID = (context & 0xfffff) << 4 */
1073 #ifdef CONFIG_PPC_KUEP
1074 oris r3, r3, SR_NX@h /* Set Nx */
1076 #ifdef CONFIG_PPC_KUAP
1077 oris r3, r3, SR_KS@h /* Set Ks */
1079 li r0,NUM_USER_SEGMENTS
1083 #ifdef CONFIG_BDI_SWITCH
1084 /* Context switch the PTE pointer for the Abatron BDI2000.
1085 * The PGDIR is passed as second argument.
1087 lis r5, abatron_pteptrs@ha
1088 stw r4, abatron_pteptrs@l + 0x4(r5)
1091 mtspr SPRN_SPRG_PGDIR, r4
1096 addi r3,r3,0x111 /* next VSID */
1097 rlwinm r3,r3,0,8,3 /* clear out any overflow from VSID field */
1098 addis r4,r4,0x1000 /* address of next segment */
1104 EMIT_BUG_ENTRY 4b,__FILE__,__LINE__,0
1106 EXPORT_SYMBOL(switch_mmu_context)
1109 * An undocumented "feature" of 604e requires that the v bit
1110 * be cleared before changing BAT values.
1112 * Also, newer IBM firmware does not clear bat3 and 4 so
1113 * this makes sure it's done.
1119 mtspr SPRN_DBAT0U,r10
1120 mtspr SPRN_DBAT0L,r10
1121 mtspr SPRN_DBAT1U,r10
1122 mtspr SPRN_DBAT1L,r10
1123 mtspr SPRN_DBAT2U,r10
1124 mtspr SPRN_DBAT2L,r10
1125 mtspr SPRN_DBAT3U,r10
1126 mtspr SPRN_DBAT3L,r10
1127 mtspr SPRN_IBAT0U,r10
1128 mtspr SPRN_IBAT0L,r10
1129 mtspr SPRN_IBAT1U,r10
1130 mtspr SPRN_IBAT1L,r10
1131 mtspr SPRN_IBAT2U,r10
1132 mtspr SPRN_IBAT2L,r10
1133 mtspr SPRN_IBAT3U,r10
1134 mtspr SPRN_IBAT3L,r10
1135 BEGIN_MMU_FTR_SECTION
1136 /* Here's a tweak: at this point, CPU setup have
1137 * not been called yet, so HIGH_BAT_EN may not be
1138 * set in HID0 for the 745x processors. However, it
1139 * seems that doesn't affect our ability to actually
1140 * write to these SPRs.
1142 mtspr SPRN_DBAT4U,r10
1143 mtspr SPRN_DBAT4L,r10
1144 mtspr SPRN_DBAT5U,r10
1145 mtspr SPRN_DBAT5L,r10
1146 mtspr SPRN_DBAT6U,r10
1147 mtspr SPRN_DBAT6L,r10
1148 mtspr SPRN_DBAT7U,r10
1149 mtspr SPRN_DBAT7L,r10
1150 mtspr SPRN_IBAT4U,r10
1151 mtspr SPRN_IBAT4L,r10
1152 mtspr SPRN_IBAT5U,r10
1153 mtspr SPRN_IBAT5L,r10
1154 mtspr SPRN_IBAT6U,r10
1155 mtspr SPRN_IBAT6L,r10
1156 mtspr SPRN_IBAT7U,r10
1157 mtspr SPRN_IBAT7L,r10
1158 END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
1167 li r3, MSR_KERNEL & ~(MSR_IR | MSR_DR)
1168 rlwinm r0, r6, 0, ~MSR_RI
1169 rlwinm r0, r0, 0, ~MSR_EE
1180 LOAD_BAT(0, r3, r4, r5)
1181 LOAD_BAT(1, r3, r4, r5)
1182 LOAD_BAT(2, r3, r4, r5)
1183 LOAD_BAT(3, r3, r4, r5)
1184 BEGIN_MMU_FTR_SECTION
1185 LOAD_BAT(4, r3, r4, r5)
1186 LOAD_BAT(5, r3, r4, r5)
1187 LOAD_BAT(6, r3, r4, r5)
1188 LOAD_BAT(7, r3, r4, r5)
1189 END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
1190 li r3, MSR_KERNEL & ~(MSR_IR | MSR_DR | MSR_RI)
1198 1: addic. r10, r10, -0x1000
1205 addi r4, r3, __after_mmu_off - _start
1207 andi. r0,r3,MSR_DR|MSR_IR /* MMU enabled? */
1217 /* We use one BAT to map up to 256M of RAM at _PAGE_OFFSET */
1219 lis r11,PAGE_OFFSET@h
1222 ori r8,r8,0x12 /* R/W access, M=1 */
1224 ori r8,r8,2 /* R/W access */
1225 #endif /* CONFIG_SMP */
1226 ori r11,r11,BL_256M<<2|0x2 /* set up BAT registers for 604 */
1228 mtspr SPRN_DBAT0L,r8 /* N.B. 6xx have valid */
1229 mtspr SPRN_DBAT0U,r11 /* bit in upper BAT register */
1230 mtspr SPRN_IBAT0L,r8
1231 mtspr SPRN_IBAT0U,r11
1235 #ifdef CONFIG_BOOTX_TEXT
1238 * setup the display bat prepared for us in prom.c
1243 addis r8,r3,disp_BAT@ha
1244 addi r8,r8,disp_BAT@l
1249 mtspr SPRN_DBAT3L,r8
1250 mtspr SPRN_DBAT3U,r11
1252 #endif /* CONFIG_BOOTX_TEXT */
1254 #ifdef CONFIG_PPC_EARLY_DEBUG_CPM
1258 mtspr SPRN_DBAT1L, r8
1261 ori r11, r11, (BL_1M << 2) | 2
1262 mtspr SPRN_DBAT1U, r11
1267 #ifdef CONFIG_PPC_EARLY_DEBUG_USBGECKO
1269 /* prepare a BAT for early io */
1270 #if defined(CONFIG_GAMECUBE)
1272 #elif defined(CONFIG_WII)
1275 #error Invalid platform for USB Gecko based early debugging.
1278 * The virtual address used must match the virtual address
1279 * associated to the fixmap entry FIX_EARLY_DEBUG_BASE.
1281 lis r11, 0xfffe /* top 128K */
1282 ori r8, r8, 0x002a /* uncached, guarded ,rw */
1283 ori r11, r11, 0x2 /* 128K, Vs=1, Vp=0 */
1284 mtspr SPRN_DBAT1L, r8
1285 mtspr SPRN_DBAT1U, r11
1290 /* Jump into the system reset for the rom.
1291 * We first disable the MMU, and then jump to the ROM reset address.
1293 * r3 is the board info structure, r4 is the location for starting.
1294 * I use this for building a small kernel that can load other kernels,
1295 * rather than trying to write or rely on a rom monitor that can tftp load.
1300 rlwinm r0,r0,0,17,15 /* clear MSR_EE in r0 */
1304 mfspr r11, SPRN_HID0
1306 ori r10,r10,HID0_ICE|HID0_DCE
1308 mtspr SPRN_HID0, r11
1310 li r5, MSR_ME|MSR_RI
1312 addis r6,r6,-KERNELBASE@h
1326 * We put a few things here that have to be page-aligned.
1327 * This stuff goes at the beginning of the data segment,
1328 * which is page-aligned.
1333 .globl empty_zero_page
1336 EXPORT_SYMBOL(empty_zero_page)
1338 .globl swapper_pg_dir
1340 .space PGD_TABLE_SIZE
1342 /* Room for two PTE pointers, usually the kernel and current user pointers
1343 * to their respective root page table.