arch/powerpc/kernel/interrupt_64.S

   1 #include <asm/asm-offsets.h>
   2 #include <asm/bug.h>
   3 #ifdef CONFIG_PPC_BOOK3S
   4 #include <asm/exception-64s.h>
   5 #else
   6 #include <asm/exception-64e.h>
   7 #endif
   8 #include <asm/feature-fixups.h>
   9 #include <asm/head-64.h>
  10 #include <asm/hw_irq.h>
  11 #include <asm/kup.h>
  12 #include <asm/mmu.h>
  13 #include <asm/ppc_asm.h>
  14 #include <asm/ptrace.h>
  15
  16         .section        ".toc","aw"
  17 SYS_CALL_TABLE:
  18         .tc sys_call_table[TC],sys_call_table
  19
  20 #ifdef CONFIG_COMPAT
  21 COMPAT_SYS_CALL_TABLE:
  22         .tc compat_sys_call_table[TC],compat_sys_call_table
  23 #endif
  24         .previous
  25
  26         .align 7
  27
  28 .macro DEBUG_SRR_VALID srr
  29 #ifdef CONFIG_PPC_RFI_SRR_DEBUG
  30         .ifc \srr,srr
  31         mfspr   r11,SPRN_SRR0
  32         ld      r12,_NIP(r1)
  33         clrrdi  r11,r11,2
  34         clrrdi  r12,r12,2
  35 100:    tdne    r11,r12
  36         EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
  37         mfspr   r11,SPRN_SRR1
  38         ld      r12,_MSR(r1)
  39 100:    tdne    r11,r12
  40         EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
  41         .else
  42         mfspr   r11,SPRN_HSRR0
  43         ld      r12,_NIP(r1)
  44         clrrdi  r11,r11,2
  45         clrrdi  r12,r12,2
  46 100:    tdne    r11,r12
  47         EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
  48         mfspr   r11,SPRN_HSRR1
  49         ld      r12,_MSR(r1)
  50 100:    tdne    r11,r12
  51         EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
  52         .endif
  53 #endif
  54 .endm
  55
  56 #ifdef CONFIG_PPC_BOOK3S
  57 .macro system_call_vectored name trapnr
  58         .globl system_call_vectored_\name
  59 system_call_vectored_\name:
  60 _ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
  61         SCV_INTERRUPT_TO_KERNEL
  62         mr      r10,r1
  63         ld      r1,PACAKSAVE(r13)
  64         std     r10,0(r1)
  65         std     r11,_NIP(r1)
  66         std     r12,_MSR(r1)
  67         std     r0,GPR0(r1)
  68         std     r10,GPR1(r1)
  69         std     r2,GPR2(r1)
  70         ld      r2,PACATOC(r13)
  71         mfcr    r12
  72         li      r11,0
  73         /* Can we avoid saving r3-r8 in common case? */
  74         std     r3,GPR3(r1)
  75         std     r4,GPR4(r1)
  76         std     r5,GPR5(r1)
  77         std     r6,GPR6(r1)
  78         std     r7,GPR7(r1)
  79         std     r8,GPR8(r1)
  80         /* Zero r9-r12, this should only be required when restoring all GPRs */
  81         std     r11,GPR9(r1)
  82         std     r11,GPR10(r1)
  83         std     r11,GPR11(r1)
  84         std     r11,GPR12(r1)
  85         std     r9,GPR13(r1)
  86         SAVE_NVGPRS(r1)
  87         std     r11,_XER(r1)
  88         std     r11,_LINK(r1)
  89         std     r11,_CTR(r1)
  90
  91         li      r11,\trapnr
  92         std     r11,_TRAP(r1)
  93         std     r12,_CCR(r1)
  94         addi    r10,r1,STACK_FRAME_OVERHEAD
  95         ld      r11,exception_marker@toc(r2)
  96         std     r11,-16(r10)            /* "regshere" marker */
  97
  98 BEGIN_FTR_SECTION
  99         HMT_MEDIUM
 100 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 101
 102         /*
 103          * scv enters with MSR[EE]=1 and is immediately considered soft-masked.
 104          * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED,
 105          * and interrupts may be masked and pending already.
 106          * system_call_exception() will call trace_hardirqs_off() which means
 107          * interrupts could already have been blocked before trace_hardirqs_off,
 108          * but this is the best we can do.
 109          */
 110
 111         /* Calling convention has r9 = orig r0, r10 = regs */
 112         mr      r9,r0
 113         bl      system_call_exception
 114
 115 .Lsyscall_vectored_\name\()_exit:
 116         addi    r4,r1,STACK_FRAME_OVERHEAD
 117         li      r5,1 /* scv */
 118         bl      syscall_exit_prepare
 119         std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
 120 .Lsyscall_vectored_\name\()_rst_start:
 121         lbz     r11,PACAIRQHAPPENED(r13)
 122         andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
 123         bne-    syscall_vectored_\name\()_restart
 124         li      r11,IRQS_ENABLED
 125         stb     r11,PACAIRQSOFTMASK(r13)
 126         li      r11,0
 127         stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
 128
 129         ld      r2,_CCR(r1)
 130         ld      r4,_NIP(r1)
 131         ld      r5,_MSR(r1)
 132
 133 BEGIN_FTR_SECTION
 134         stdcx.  r0,0,r1                 /* to clear the reservation */
 135 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
 136
 137 BEGIN_FTR_SECTION
 138         HMT_MEDIUM_LOW
 139 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 140
 141         cmpdi   r3,0
 142         bne     .Lsyscall_vectored_\name\()_restore_regs
 143
 144         /* rfscv returns with LR->NIA and CTR->MSR */
 145         mtlr    r4
 146         mtctr   r5
 147
 148         /* Could zero these as per ABI, but we may consider a stricter ABI
 149          * which preserves these if libc implementations can benefit, so
 150          * restore them for now until further measurement is done. */
 151         ld      r0,GPR0(r1)
 152         ld      r4,GPR4(r1)
 153         ld      r5,GPR5(r1)
 154         ld      r6,GPR6(r1)
 155         ld      r7,GPR7(r1)
 156         ld      r8,GPR8(r1)
 157         /* Zero volatile regs that may contain sensitive kernel data */
 158         li      r9,0
 159         li      r10,0
 160         li      r11,0
 161         li      r12,0
 162         mtspr   SPRN_XER,r0
 163
 164         /*
 165          * We don't need to restore AMR on the way back to userspace for KUAP.
 166          * The value of AMR only matters while we're in the kernel.
 167          */
 168         mtcr    r2
 169         REST_GPRS(2, 3, r1)
 170         REST_GPR(13, r1)
 171         REST_GPR(1, r1)
 172         RFSCV_TO_USER
 173         b       .       /* prevent speculative execution */
 174
 175 .Lsyscall_vectored_\name\()_restore_regs:
 176         mtspr   SPRN_SRR0,r4
 177         mtspr   SPRN_SRR1,r5
 178
 179         ld      r3,_CTR(r1)
 180         ld      r4,_LINK(r1)
 181         ld      r5,_XER(r1)
 182
 183         REST_NVGPRS(r1)
 184         ld      r0,GPR0(r1)
 185         mtcr    r2
 186         mtctr   r3
 187         mtlr    r4
 188         mtspr   SPRN_XER,r5
 189         REST_GPRS(2, 13, r1)
 190         REST_GPR(1, r1)
 191         RFI_TO_USER
 192 .Lsyscall_vectored_\name\()_rst_end:
 193
 194 syscall_vectored_\name\()_restart:
 195 _ASM_NOKPROBE_SYMBOL(syscall_vectored_\name\()_restart)
 196         GET_PACA(r13)
 197         ld      r1,PACA_EXIT_SAVE_R1(r13)
 198         ld      r2,PACATOC(r13)
 199         ld      r3,RESULT(r1)
 200         addi    r4,r1,STACK_FRAME_OVERHEAD
 201         li      r11,IRQS_ALL_DISABLED
 202         stb     r11,PACAIRQSOFTMASK(r13)
 203         bl      syscall_exit_restart
 204         std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
 205         b       .Lsyscall_vectored_\name\()_rst_start
 206 1:
 207
 208 SOFT_MASK_TABLE(.Lsyscall_vectored_\name\()_rst_start, 1b)
 209 RESTART_TABLE(.Lsyscall_vectored_\name\()_rst_start, .Lsyscall_vectored_\name\()_rst_end, syscall_vectored_\name\()_restart)
 210
 211 .endm
 212
 213 system_call_vectored common 0x3000
 214
 215 /*
 216  * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
 217  * which is tested by system_call_exception when r0 is -1 (as set by vector
 218  * entry code).
 219  */
 220 system_call_vectored sigill 0x7ff0
 221
 222
 223 /*
 224  * Entered via kernel return set up by kernel/sstep.c, must match entry regs
 225  */
 226         .globl system_call_vectored_emulate
 227 system_call_vectored_emulate:
 228 _ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
 229         li      r10,IRQS_ALL_DISABLED
 230         stb     r10,PACAIRQSOFTMASK(r13)
 231         b       system_call_vectored_common
 232 #endif /* CONFIG_PPC_BOOK3S */
 233
 234         .balign IFETCH_ALIGN_BYTES
 235         .globl system_call_common_real
 236 system_call_common_real:
 237 _ASM_NOKPROBE_SYMBOL(system_call_common_real)
 238         ld      r10,PACAKMSR(r13)       /* get MSR value for kernel */
 239         mtmsrd  r10
 240
 241         .balign IFETCH_ALIGN_BYTES
 242         .globl system_call_common
 243 system_call_common:
 244 _ASM_NOKPROBE_SYMBOL(system_call_common)
 245         mr      r10,r1
 246         ld      r1,PACAKSAVE(r13)
 247         std     r10,0(r1)
 248         std     r11,_NIP(r1)
 249         std     r12,_MSR(r1)
 250         std     r0,GPR0(r1)
 251         std     r10,GPR1(r1)
 252         std     r2,GPR2(r1)
 253 #ifdef CONFIG_PPC_FSL_BOOK3E
 254 START_BTB_FLUSH_SECTION
 255         BTB_FLUSH(r10)
 256 END_BTB_FLUSH_SECTION
 257 #endif
 258         ld      r2,PACATOC(r13)
 259         mfcr    r12
 260         li      r11,0
 261         /* Can we avoid saving r3-r8 in common case? */
 262         std     r3,GPR3(r1)
 263         std     r4,GPR4(r1)
 264         std     r5,GPR5(r1)
 265         std     r6,GPR6(r1)
 266         std     r7,GPR7(r1)
 267         std     r8,GPR8(r1)
 268         /* Zero r9-r12, this should only be required when restoring all GPRs */
 269         std     r11,GPR9(r1)
 270         std     r11,GPR10(r1)
 271         std     r11,GPR11(r1)
 272         std     r11,GPR12(r1)
 273         std     r9,GPR13(r1)
 274         SAVE_NVGPRS(r1)
 275         std     r11,_XER(r1)
 276         std     r11,_CTR(r1)
 277         mflr    r10
 278
 279         /*
 280          * This clears CR0.SO (bit 28), which is the error indication on
 281          * return from this system call.
 282          */
 283         rldimi  r12,r11,28,(63-28)
 284         li      r11,0xc00
 285         std     r10,_LINK(r1)
 286         std     r11,_TRAP(r1)
 287         std     r12,_CCR(r1)
 288         addi    r10,r1,STACK_FRAME_OVERHEAD
 289         ld      r11,exception_marker@toc(r2)
 290         std     r11,-16(r10)            /* "regshere" marker */
 291
 292 #ifdef CONFIG_PPC_BOOK3S
 293         li      r11,1
 294         stb     r11,PACASRR_VALID(r13)
 295 #endif
 296
 297         /*
 298          * We always enter kernel from userspace with irq soft-mask enabled and
 299          * nothing pending. system_call_exception() will call
 300          * trace_hardirqs_off().
 301          */
 302         li      r11,IRQS_ALL_DISABLED
 303         stb     r11,PACAIRQSOFTMASK(r13)
 304 #ifdef CONFIG_PPC_BOOK3S
 305         li      r12,-1 /* Set MSR_EE and MSR_RI */
 306         mtmsrd  r12,1
 307 #else
 308         wrteei  1
 309 #endif
 310
 311         /* Calling convention has r9 = orig r0, r10 = regs */
 312         mr      r9,r0
 313         bl      system_call_exception
 314
 315 .Lsyscall_exit:
 316         addi    r4,r1,STACK_FRAME_OVERHEAD
 317         li      r5,0 /* !scv */
 318         bl      syscall_exit_prepare
 319         std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
 320 #ifdef CONFIG_PPC_BOOK3S
 321 .Lsyscall_rst_start:
 322         lbz     r11,PACAIRQHAPPENED(r13)
 323         andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
 324         bne-    syscall_restart
 325 #endif
 326         li      r11,IRQS_ENABLED
 327         stb     r11,PACAIRQSOFTMASK(r13)
 328         li      r11,0
 329         stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
 330
 331         ld      r2,_CCR(r1)
 332         ld      r6,_LINK(r1)
 333         mtlr    r6
 334
 335 #ifdef CONFIG_PPC_BOOK3S
 336         lbz     r4,PACASRR_VALID(r13)
 337         cmpdi   r4,0
 338         bne     1f
 339         li      r4,0
 340         stb     r4,PACASRR_VALID(r13)
 341 #endif
 342         ld      r4,_NIP(r1)
 343         ld      r5,_MSR(r1)
 344         mtspr   SPRN_SRR0,r4
 345         mtspr   SPRN_SRR1,r5
 346 1:
 347         DEBUG_SRR_VALID srr
 348
 349 BEGIN_FTR_SECTION
 350         stdcx.  r0,0,r1                 /* to clear the reservation */
 351 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
 352
 353         cmpdi   r3,0
 354         bne     .Lsyscall_restore_regs
 355         /* Zero volatile regs that may contain sensitive kernel data */
 356         li      r0,0
 357         li      r4,0
 358         li      r5,0
 359         li      r6,0
 360         li      r7,0
 361         li      r8,0
 362         li      r9,0
 363         li      r10,0
 364         li      r11,0
 365         li      r12,0
 366         mtctr   r0
 367         mtspr   SPRN_XER,r0
 368 .Lsyscall_restore_regs_cont:
 369
 370 BEGIN_FTR_SECTION
 371         HMT_MEDIUM_LOW
 372 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 373
 374         /*
 375          * We don't need to restore AMR on the way back to userspace for KUAP.
 376          * The value of AMR only matters while we're in the kernel.
 377          */
 378         mtcr    r2
 379         REST_GPRS(2, 3, r1)
 380         REST_GPR(13, r1)
 381         REST_GPR(1, r1)
 382         RFI_TO_USER
 383         b       .       /* prevent speculative execution */
 384
 385 .Lsyscall_restore_regs:
 386         ld      r3,_CTR(r1)
 387         ld      r4,_XER(r1)
 388         REST_NVGPRS(r1)
 389         mtctr   r3
 390         mtspr   SPRN_XER,r4
 391         ld      r0,GPR0(r1)
 392         REST_GPRS(4, 12, r1)
 393         b       .Lsyscall_restore_regs_cont
 394 .Lsyscall_rst_end:
 395
 396 #ifdef CONFIG_PPC_BOOK3S
 397 syscall_restart:
 398 _ASM_NOKPROBE_SYMBOL(syscall_restart)
 399         GET_PACA(r13)
 400         ld      r1,PACA_EXIT_SAVE_R1(r13)
 401         ld      r2,PACATOC(r13)
 402         ld      r3,RESULT(r1)
 403         addi    r4,r1,STACK_FRAME_OVERHEAD
 404         li      r11,IRQS_ALL_DISABLED
 405         stb     r11,PACAIRQSOFTMASK(r13)
 406         bl      syscall_exit_restart
 407         std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
 408         b       .Lsyscall_rst_start
 409 1:
 410
 411 SOFT_MASK_TABLE(.Lsyscall_rst_start, 1b)
 412 RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)
 413 #endif
 414
 415         /*
 416          * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
 417          * touched, no exit work created, then this can be used.
 418          */
 419         .balign IFETCH_ALIGN_BYTES
 420         .globl fast_interrupt_return_srr
 421 fast_interrupt_return_srr:
 422 _ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
 423         kuap_check_amr r3, r4
 424         ld      r5,_MSR(r1)
 425         andi.   r0,r5,MSR_PR
 426 #ifdef CONFIG_PPC_BOOK3S
 427         beq     1f
 428         kuap_user_restore r3, r4
 429         b       .Lfast_user_interrupt_return_srr
 430 1:      kuap_kernel_restore r3, r4
 431         andi.   r0,r5,MSR_RI
 432         li      r3,0 /* 0 return value, no EMULATE_STACK_STORE */
 433         bne+    .Lfast_kernel_interrupt_return_srr
 434         addi    r3,r1,STACK_FRAME_OVERHEAD
 435         bl      unrecoverable_exception
 436         b       . /* should not get here */
 437 #else
 438         bne     .Lfast_user_interrupt_return_srr
 439         b       .Lfast_kernel_interrupt_return_srr
 440 #endif
 441
 442 .macro interrupt_return_macro srr
 443         .balign IFETCH_ALIGN_BYTES
 444         .globl interrupt_return_\srr
 445 interrupt_return_\srr\():
 446 _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
 447         ld      r4,_MSR(r1)
 448         andi.   r0,r4,MSR_PR
 449         beq     interrupt_return_\srr\()_kernel
 450 interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */
 451 _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
 452         addi    r3,r1,STACK_FRAME_OVERHEAD
 453         bl      interrupt_exit_user_prepare
 454         cmpdi   r3,0
 455         bne-    .Lrestore_nvgprs_\srr
 456 .Lrestore_nvgprs_\srr\()_cont:
 457         std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
 458 #ifdef CONFIG_PPC_BOOK3S
 459 .Linterrupt_return_\srr\()_user_rst_start:
 460         lbz     r11,PACAIRQHAPPENED(r13)
 461         andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
 462         bne-    interrupt_return_\srr\()_user_restart
 463 #endif
 464         li      r11,IRQS_ENABLED
 465         stb     r11,PACAIRQSOFTMASK(r13)
 466         li      r11,0
 467         stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
 468
 469 .Lfast_user_interrupt_return_\srr\():
 470 #ifdef CONFIG_PPC_BOOK3S
 471         .ifc \srr,srr
 472         lbz     r4,PACASRR_VALID(r13)
 473         .else
 474         lbz     r4,PACAHSRR_VALID(r13)
 475         .endif
 476         cmpdi   r4,0
 477         li      r4,0
 478         bne     1f
 479 #endif
 480         ld      r11,_NIP(r1)
 481         ld      r12,_MSR(r1)
 482         .ifc \srr,srr
 483         mtspr   SPRN_SRR0,r11
 484         mtspr   SPRN_SRR1,r12
 485 1:
 486 #ifdef CONFIG_PPC_BOOK3S
 487         stb     r4,PACASRR_VALID(r13)
 488 #endif
 489         .else
 490         mtspr   SPRN_HSRR0,r11
 491         mtspr   SPRN_HSRR1,r12
 492 1:
 493 #ifdef CONFIG_PPC_BOOK3S
 494         stb     r4,PACAHSRR_VALID(r13)
 495 #endif
 496         .endif
 497         DEBUG_SRR_VALID \srr
 498
 499 #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
 500         lbz     r4,PACAIRQSOFTMASK(r13)
 501         tdnei   r4,IRQS_ENABLED
 502 #endif
 503
 504 BEGIN_FTR_SECTION
 505         ld      r10,_PPR(r1)
 506         mtspr   SPRN_PPR,r10
 507 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 508
 509 BEGIN_FTR_SECTION
 510         stdcx.  r0,0,r1         /* to clear the reservation */
 511 FTR_SECTION_ELSE
 512         ldarx   r0,0,r1
 513 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
 514
 515         ld      r3,_CCR(r1)
 516         ld      r4,_LINK(r1)
 517         ld      r5,_CTR(r1)
 518         ld      r6,_XER(r1)
 519         li      r0,0
 520
 521         REST_GPRS(7, 13, r1)
 522
 523         mtcr    r3
 524         mtlr    r4
 525         mtctr   r5
 526         mtspr   SPRN_XER,r6
 527
 528         REST_GPRS(2, 6, r1)
 529         REST_GPR(0, r1)
 530         REST_GPR(1, r1)
 531         .ifc \srr,srr
 532         RFI_TO_USER
 533         .else
 534         HRFI_TO_USER
 535         .endif
 536         b       .       /* prevent speculative execution */
 537 .Linterrupt_return_\srr\()_user_rst_end:
 538
 539 .Lrestore_nvgprs_\srr\():
 540         REST_NVGPRS(r1)
 541         b       .Lrestore_nvgprs_\srr\()_cont
 542
 543 #ifdef CONFIG_PPC_BOOK3S
 544 interrupt_return_\srr\()_user_restart:
 545 _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user_restart)
 546         GET_PACA(r13)
 547         ld      r1,PACA_EXIT_SAVE_R1(r13)
 548         ld      r2,PACATOC(r13)
 549         addi    r3,r1,STACK_FRAME_OVERHEAD
 550         li      r11,IRQS_ALL_DISABLED
 551         stb     r11,PACAIRQSOFTMASK(r13)
 552         bl      interrupt_exit_user_restart
 553         std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
 554         b       .Linterrupt_return_\srr\()_user_rst_start
 555 1:
 556
 557 SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_user_rst_start, 1b)
 558 RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)
 559 #endif
 560
 561         .balign IFETCH_ALIGN_BYTES
 562 interrupt_return_\srr\()_kernel:
 563 _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel)
 564         addi    r3,r1,STACK_FRAME_OVERHEAD
 565         bl      interrupt_exit_kernel_prepare
 566
 567         std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
 568 .Linterrupt_return_\srr\()_kernel_rst_start:
 569         ld      r11,SOFTE(r1)
 570         cmpwi   r11,IRQS_ENABLED
 571         stb     r11,PACAIRQSOFTMASK(r13)
 572         bne     1f
 573 #ifdef CONFIG_PPC_BOOK3S
 574         lbz     r11,PACAIRQHAPPENED(r13)
 575         andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
 576         bne-    interrupt_return_\srr\()_kernel_restart
 577 #endif
 578         li      r11,0
 579         stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
 580 1:
 581
 582 .Lfast_kernel_interrupt_return_\srr\():
 583         cmpdi   cr1,r3,0
 584 #ifdef CONFIG_PPC_BOOK3S
 585         .ifc \srr,srr
 586         lbz     r4,PACASRR_VALID(r13)
 587         .else
 588         lbz     r4,PACAHSRR_VALID(r13)
 589         .endif
 590         cmpdi   r4,0
 591         li      r4,0
 592         bne     1f
 593 #endif
 594         ld      r11,_NIP(r1)
 595         ld      r12,_MSR(r1)
 596         .ifc \srr,srr
 597         mtspr   SPRN_SRR0,r11
 598         mtspr   SPRN_SRR1,r12
 599 1:
 600 #ifdef CONFIG_PPC_BOOK3S
 601         stb     r4,PACASRR_VALID(r13)
 602 #endif
 603         .else
 604         mtspr   SPRN_HSRR0,r11
 605         mtspr   SPRN_HSRR1,r12
 606 1:
 607 #ifdef CONFIG_PPC_BOOK3S
 608         stb     r4,PACAHSRR_VALID(r13)
 609 #endif
 610         .endif
 611         DEBUG_SRR_VALID \srr
 612
 613 BEGIN_FTR_SECTION
 614         stdcx.  r0,0,r1         /* to clear the reservation */
 615 FTR_SECTION_ELSE
 616         ldarx   r0,0,r1
 617 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
 618
 619         ld      r3,_LINK(r1)
 620         ld      r4,_CTR(r1)
 621         ld      r5,_XER(r1)
 622         ld      r6,_CCR(r1)
 623         li      r0,0
 624
 625         REST_GPRS(7, 12, r1)
 626
 627         mtlr    r3
 628         mtctr   r4
 629         mtspr   SPRN_XER,r5
 630
 631         /*
 632          * Leaving a stale exception_marker on the stack can confuse
 633          * the reliable stack unwinder later on. Clear it.
 634          */
 635         std     r0,STACK_FRAME_OVERHEAD-16(r1)
 636
 637         REST_GPRS(2, 5, r1)
 638
 639         bne-    cr1,1f /* emulate stack store */
 640         mtcr    r6
 641         REST_GPR(6, r1)
 642         REST_GPR(0, r1)
 643         REST_GPR(1, r1)
 644         .ifc \srr,srr
 645         RFI_TO_KERNEL
 646         .else
 647         HRFI_TO_KERNEL
 648         .endif
 649         b       .       /* prevent speculative execution */
 650
 651 1:      /*
 652          * Emulate stack store with update. New r1 value was already calculated
 653          * and updated in our interrupt regs by emulate_loadstore, but we can't
 654          * store the previous value of r1 to the stack before re-loading our
 655          * registers from it, otherwise they could be clobbered.  Use
 656          * PACA_EXGEN as temporary storage to hold the store data, as
 657          * interrupts are disabled here so it won't be clobbered.
 658          */
 659         mtcr    r6
 660         std     r9,PACA_EXGEN+0(r13)
 661         addi    r9,r1,INT_FRAME_SIZE /* get original r1 */
 662         REST_GPR(6, r1)
 663         REST_GPR(0, r1)
 664         REST_GPR(1, r1)
 665         std     r9,0(r1) /* perform store component of stdu */
 666         ld      r9,PACA_EXGEN+0(r13)
 667
 668         .ifc \srr,srr
 669         RFI_TO_KERNEL
 670         .else
 671         HRFI_TO_KERNEL
 672         .endif
 673         b       .       /* prevent speculative execution */
 674 .Linterrupt_return_\srr\()_kernel_rst_end:
 675
 676 #ifdef CONFIG_PPC_BOOK3S
 677 interrupt_return_\srr\()_kernel_restart:
 678 _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel_restart)
 679         GET_PACA(r13)
 680         ld      r1,PACA_EXIT_SAVE_R1(r13)
 681         ld      r2,PACATOC(r13)
 682         addi    r3,r1,STACK_FRAME_OVERHEAD
 683         li      r11,IRQS_ALL_DISABLED
 684         stb     r11,PACAIRQSOFTMASK(r13)
 685         bl      interrupt_exit_kernel_restart
 686         std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
 687         b       .Linterrupt_return_\srr\()_kernel_rst_start
 688 1:
 689
 690 SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, 1b)
 691 RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)
 692 #endif
 693
 694 .endm
 695
 696 interrupt_return_macro srr
 697 #ifdef CONFIG_PPC_BOOK3S
 698 interrupt_return_macro hsrr
 699
 700         .globl __end_soft_masked
 701 __end_soft_masked:
 702 DEFINE_FIXED_SYMBOL(__end_soft_masked)
 703 #endif /* CONFIG_PPC_BOOK3S */
 704
 705 #ifdef CONFIG_PPC_BOOK3S
 706 _GLOBAL(ret_from_fork_scv)
 707         bl      schedule_tail
 708         REST_NVGPRS(r1)
 709         li      r3,0    /* fork() return value */
 710         b       .Lsyscall_vectored_common_exit
 711 #endif
 712
 713 _GLOBAL(ret_from_fork)
 714         bl      schedule_tail
 715         REST_NVGPRS(r1)
 716         li      r3,0    /* fork() return value */
 717         b       .Lsyscall_exit
 718
 719 _GLOBAL(ret_from_kernel_thread)
 720         bl      schedule_tail
 721         REST_NVGPRS(r1)
 722         mtctr   r14
 723         mr      r3,r15
 724 #ifdef PPC64_ELF_ABI_v2
 725         mr      r12,r14
 726 #endif
 727         bctrl
 728         li      r3,0
 729         b       .Lsyscall_exit