2 * PARISC TLB and cache flushing support
3 * Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 * Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 * Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2, or (at your option)
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 * should only use index and base registers that are not shadowed,
25 * so that the fast path emulation in the non access miss handler
36 #include <asm/assembly.h>
37 #include <asm/pgtable.h>
38 #include <asm/cache.h>
40 #include <linux/linkage.h>
45 ENTRY_CFI(flush_tlb_all_local)
51 * The pitlbe and pdtlbe instructions should only be used to
52 * flush the entire tlb. Also, there needs to be no intervening
53 * tlb operations, e.g. tlb misses, so the operation needs
54 * to happen in real mode with all interruptions disabled.
57 /* pcxt_ssm_bug - relied upon translation! PA 2.0 Arch. F-4 and F-5 */
58 rsm PSW_SM_I, %r19 /* save I-bit state */
66 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
67 mtctl %r0, %cr17 /* Clear IIASQ tail */
68 mtctl %r0, %cr17 /* Clear IIASQ head */
69 mtctl %r1, %cr18 /* IIAOQ head */
71 mtctl %r1, %cr18 /* IIAOQ tail */
72 load32 REAL_MODE_PSW, %r1
77 1: load32 PA(cache_info), %r1
79 /* Flush Instruction Tlb */
81 LDREG ITLB_SID_BASE(%r1), %r20
82 LDREG ITLB_SID_STRIDE(%r1), %r21
83 LDREG ITLB_SID_COUNT(%r1), %r22
84 LDREG ITLB_OFF_BASE(%r1), %arg0
85 LDREG ITLB_OFF_STRIDE(%r1), %arg1
86 LDREG ITLB_OFF_COUNT(%r1), %arg2
87 LDREG ITLB_LOOP(%r1), %arg3
89 addib,COND(=) -1, %arg3, fitoneloop /* Preadjust and test */
90 movb,<,n %arg3, %r31, fitdone /* If loop < 0, skip */
91 copy %arg0, %r28 /* Init base addr */
93 fitmanyloop: /* Loop if LOOP >= 2 */
95 add %r21, %r20, %r20 /* increment space */
96 copy %arg2, %r29 /* Init middle loop count */
98 fitmanymiddle: /* Loop if LOOP >= 2 */
99 addib,COND(>) -1, %r31, fitmanymiddle /* Adjusted inner loop decr */
100 pitlbe %r0(%sr1, %r28)
101 pitlbe,m %arg1(%sr1, %r28) /* Last pitlbe and addr adjust */
102 addib,COND(>) -1, %r29, fitmanymiddle /* Middle loop decr */
103 copy %arg3, %r31 /* Re-init inner loop count */
105 movb,tr %arg0, %r28, fitmanyloop /* Re-init base addr */
106 addib,COND(<=),n -1, %r22, fitdone /* Outer loop count decr */
108 fitoneloop: /* Loop if LOOP = 1 */
110 copy %arg0, %r28 /* init base addr */
111 copy %arg2, %r29 /* init middle loop count */
113 fitonemiddle: /* Loop if LOOP = 1 */
114 addib,COND(>) -1, %r29, fitonemiddle /* Middle loop count decr */
115 pitlbe,m %arg1(%sr1, %r28) /* pitlbe for one loop */
117 addib,COND(>) -1, %r22, fitoneloop /* Outer loop count decr */
118 add %r21, %r20, %r20 /* increment space */
124 LDREG DTLB_SID_BASE(%r1), %r20
125 LDREG DTLB_SID_STRIDE(%r1), %r21
126 LDREG DTLB_SID_COUNT(%r1), %r22
127 LDREG DTLB_OFF_BASE(%r1), %arg0
128 LDREG DTLB_OFF_STRIDE(%r1), %arg1
129 LDREG DTLB_OFF_COUNT(%r1), %arg2
130 LDREG DTLB_LOOP(%r1), %arg3
132 addib,COND(=) -1, %arg3, fdtoneloop /* Preadjust and test */
133 movb,<,n %arg3, %r31, fdtdone /* If loop < 0, skip */
134 copy %arg0, %r28 /* Init base addr */
136 fdtmanyloop: /* Loop if LOOP >= 2 */
138 add %r21, %r20, %r20 /* increment space */
139 copy %arg2, %r29 /* Init middle loop count */
141 fdtmanymiddle: /* Loop if LOOP >= 2 */
142 addib,COND(>) -1, %r31, fdtmanymiddle /* Adjusted inner loop decr */
143 pdtlbe %r0(%sr1, %r28)
144 pdtlbe,m %arg1(%sr1, %r28) /* Last pdtlbe and addr adjust */
145 addib,COND(>) -1, %r29, fdtmanymiddle /* Middle loop decr */
146 copy %arg3, %r31 /* Re-init inner loop count */
148 movb,tr %arg0, %r28, fdtmanyloop /* Re-init base addr */
149 addib,COND(<=),n -1, %r22,fdtdone /* Outer loop count decr */
151 fdtoneloop: /* Loop if LOOP = 1 */
153 copy %arg0, %r28 /* init base addr */
154 copy %arg2, %r29 /* init middle loop count */
156 fdtonemiddle: /* Loop if LOOP = 1 */
157 addib,COND(>) -1, %r29, fdtonemiddle /* Middle loop count decr */
158 pdtlbe,m %arg1(%sr1, %r28) /* pdtlbe for one loop */
160 addib,COND(>) -1, %r22, fdtoneloop /* Outer loop count decr */
161 add %r21, %r20, %r20 /* increment space */
166 * Switch back to virtual mode
177 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
178 mtctl %r0, %cr17 /* Clear IIASQ tail */
179 mtctl %r0, %cr17 /* Clear IIASQ head */
180 mtctl %r1, %cr18 /* IIAOQ head */
182 mtctl %r1, %cr18 /* IIAOQ tail */
183 load32 KERNEL_PSW, %r1
184 or %r1, %r19, %r1 /* I-bit to state on entry */
185 mtctl %r1, %ipsw /* restore I-bit (entire PSW) */
194 ENDPROC_CFI(flush_tlb_all_local)
196 .import cache_info,data
198 ENTRY_CFI(flush_instruction_cache_local)
203 load32 cache_info, %r1
205 /* Flush Instruction Cache */
207 LDREG ICACHE_BASE(%r1), %arg0
208 LDREG ICACHE_STRIDE(%r1), %arg1
209 LDREG ICACHE_COUNT(%r1), %arg2
210 LDREG ICACHE_LOOP(%r1), %arg3
211 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
213 addib,COND(=) -1, %arg3, fioneloop /* Preadjust and test */
214 movb,<,n %arg3, %r31, fisync /* If loop < 0, do sync */
216 fimanyloop: /* Loop if LOOP >= 2 */
217 addib,COND(>) -1, %r31, fimanyloop /* Adjusted inner loop decr */
218 fice %r0(%sr1, %arg0)
219 fice,m %arg1(%sr1, %arg0) /* Last fice and addr adjust */
220 movb,tr %arg3, %r31, fimanyloop /* Re-init inner loop count */
221 addib,COND(<=),n -1, %arg2, fisync /* Outer loop decr */
223 fioneloop: /* Loop if LOOP = 1 */
224 /* Some implementations may flush with a single fice instruction */
225 cmpib,COND(>>=),n 15, %arg2, fioneloop2
228 fice,m %arg1(%sr1, %arg0)
229 fice,m %arg1(%sr1, %arg0)
230 fice,m %arg1(%sr1, %arg0)
231 fice,m %arg1(%sr1, %arg0)
232 fice,m %arg1(%sr1, %arg0)
233 fice,m %arg1(%sr1, %arg0)
234 fice,m %arg1(%sr1, %arg0)
235 fice,m %arg1(%sr1, %arg0)
236 fice,m %arg1(%sr1, %arg0)
237 fice,m %arg1(%sr1, %arg0)
238 fice,m %arg1(%sr1, %arg0)
239 fice,m %arg1(%sr1, %arg0)
240 fice,m %arg1(%sr1, %arg0)
241 fice,m %arg1(%sr1, %arg0)
242 fice,m %arg1(%sr1, %arg0)
243 addib,COND(>) -16, %arg2, fioneloop1
244 fice,m %arg1(%sr1, %arg0)
247 cmpb,COND(=),n %arg2, %r0, fisync /* Predict branch taken */
250 addib,COND(>) -1, %arg2, fioneloop2 /* Outer loop count decr */
251 fice,m %arg1(%sr1, %arg0) /* Fice for one loop */
255 mtsm %r22 /* restore I-bit */
261 ENDPROC_CFI(flush_instruction_cache_local)
264 .import cache_info, data
265 ENTRY_CFI(flush_data_cache_local)
270 load32 cache_info, %r1
272 /* Flush Data Cache */
274 LDREG DCACHE_BASE(%r1), %arg0
275 LDREG DCACHE_STRIDE(%r1), %arg1
276 LDREG DCACHE_COUNT(%r1), %arg2
277 LDREG DCACHE_LOOP(%r1), %arg3
278 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
280 addib,COND(=) -1, %arg3, fdoneloop /* Preadjust and test */
281 movb,<,n %arg3, %r31, fdsync /* If loop < 0, do sync */
283 fdmanyloop: /* Loop if LOOP >= 2 */
284 addib,COND(>) -1, %r31, fdmanyloop /* Adjusted inner loop decr */
285 fdce %r0(%sr1, %arg0)
286 fdce,m %arg1(%sr1, %arg0) /* Last fdce and addr adjust */
287 movb,tr %arg3, %r31, fdmanyloop /* Re-init inner loop count */
288 addib,COND(<=),n -1, %arg2, fdsync /* Outer loop decr */
290 fdoneloop: /* Loop if LOOP = 1 */
291 /* Some implementations may flush with a single fdce instruction */
292 cmpib,COND(>>=),n 15, %arg2, fdoneloop2
295 fdce,m %arg1(%sr1, %arg0)
296 fdce,m %arg1(%sr1, %arg0)
297 fdce,m %arg1(%sr1, %arg0)
298 fdce,m %arg1(%sr1, %arg0)
299 fdce,m %arg1(%sr1, %arg0)
300 fdce,m %arg1(%sr1, %arg0)
301 fdce,m %arg1(%sr1, %arg0)
302 fdce,m %arg1(%sr1, %arg0)
303 fdce,m %arg1(%sr1, %arg0)
304 fdce,m %arg1(%sr1, %arg0)
305 fdce,m %arg1(%sr1, %arg0)
306 fdce,m %arg1(%sr1, %arg0)
307 fdce,m %arg1(%sr1, %arg0)
308 fdce,m %arg1(%sr1, %arg0)
309 fdce,m %arg1(%sr1, %arg0)
310 addib,COND(>) -16, %arg2, fdoneloop1
311 fdce,m %arg1(%sr1, %arg0)
314 cmpb,COND(=),n %arg2, %r0, fdsync /* Predict branch taken */
317 addib,COND(>) -1, %arg2, fdoneloop2 /* Outer loop count decr */
318 fdce,m %arg1(%sr1, %arg0) /* Fdce for one loop */
323 mtsm %r22 /* restore I-bit */
329 ENDPROC_CFI(flush_data_cache_local)
333 /* Macros to serialize TLB purge operations on SMP. */
335 .macro tlb_lock la,flags,tmp
337 #if __PA_LDCW_ALIGNMENT > 4
338 load32 pa_tlb_lock + __PA_LDCW_ALIGNMENT-1, \la
339 depi 0,31,__PA_LDCW_ALIGN_ORDER, \la
341 load32 pa_tlb_lock, \la
354 .macro tlb_unlock la,flags,tmp
363 /* Clear page using kernel mapping. */
365 ENTRY_CFI(clear_page_asm)
372 /* Unroll the loop. */
373 ldi (PAGE_SIZE / 128), %r1
393 /* Note reverse branch hint for addib is taken. */
394 addib,COND(>),n -1, %r1, 1b
400 * Note that until (if) we start saving the full 64-bit register
401 * values on interrupt, we can't use std on a 32 bit kernel.
403 ldi (PAGE_SIZE / 64), %r1
423 addib,COND(>),n -1, %r1, 1b
431 ENDPROC_CFI(clear_page_asm)
433 /* Copy page using kernel mapping. */
435 ENTRY_CFI(copy_page_asm)
441 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
442 * Unroll the loop by hand and arrange insn appropriately.
443 * Prefetch doesn't improve performance on rp3440.
444 * GCC probably can do this just as well...
447 ldi (PAGE_SIZE / 128), %r1
491 /* Note reverse branch hint for addib is taken. */
492 addib,COND(>),n -1, %r1, 1b
498 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
499 * bundles (very restricted rules for bundling).
500 * Note that until (if) we start saving
501 * the full 64 bit register values on interrupt, we can't
502 * use ldd/std on a 32 bit kernel.
505 ldi (PAGE_SIZE / 64), %r1
541 addib,COND(>),n -1, %r1, 1b
549 ENDPROC_CFI(copy_page_asm)
552 * NOTE: Code in clear_user_page has a hard coded dependency on the
553 * maximum alias boundary being 4 Mb. We've been assured by the
554 * parisc chip designers that there will not ever be a parisc
555 * chip with a larger alias boundary (Never say never :-) ).
557 * Subtle: the dtlb miss handlers support the temp alias region by
558 * "knowing" that if a dtlb miss happens within the temp alias
559 * region it must have occurred while in clear_user_page. Since
560 * this routine makes use of processor local translations, we
561 * don't want to insert them into the kernel page table. Instead,
562 * we load up some general registers (they need to be registers
563 * which aren't shadowed) with the physical page numbers (preshifted
564 * for tlb insertion) needed to insert the translations. When we
565 * miss on the translation, the dtlb miss handler inserts the
566 * translation into the tlb using these values:
568 * %r26 physical page (shifted for tlb insert) of "to" translation
569 * %r23 physical page (shifted for tlb insert) of "from" translation
572 /* Drop prot bits and convert to page addr for iitlbt and idtlbt */
573 #define PAGE_ADD_SHIFT (PAGE_SHIFT-12)
574 .macro convert_phys_for_tlb_insert20 phys
575 extrd,u \phys, 56-PAGE_ADD_SHIFT, 32-PAGE_ADD_SHIFT, \phys
576 #if _PAGE_SIZE_ENCODING_DEFAULT
577 depdi _PAGE_SIZE_ENCODING_DEFAULT, 63, (63-58), \phys
582 * copy_user_page_asm() performs a page copy using mappings
583 * equivalent to the user page mappings. It can be used to
584 * implement copy_user_page() but unfortunately both the `from'
585 * and `to' pages need to be flushed through mappings equivalent
586 * to the user mappings after the copy because the kernel accesses
587 * the `from' page through the kmap kernel mapping and the `to'
588 * page needs to be flushed since code can be copied. As a
589 * result, this implementation is less efficient than the simpler
590 * copy using the kernel mapping. It only needs the `from' page
591 * to flushed via the user mapping. The kunmap routines handle
592 * the flushes needed for the kernel mapping.
594 * I'm still keeping this around because it may be possible to
595 * use it if more information is passed into copy_user_page().
596 * Have to do some measurements to see if it is worthwhile to
597 * lobby for such a change.
601 ENTRY_CFI(copy_user_page_asm)
606 /* Convert virtual `to' and `from' addresses to physical addresses.
607 Move `from' physical address to non shadowed register. */
608 ldil L%(__PAGE_OFFSET), %r1
612 ldil L%(TMPALIAS_MAP_START), %r28
614 #if (TMPALIAS_MAP_START >= 0x80000000)
615 depdi 0, 31,32, %r28 /* clear any sign extension */
617 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
618 convert_phys_for_tlb_insert20 %r23 /* convert phys addr to tlb insert format */
619 depd %r24,63,22, %r28 /* Form aliased virtual address 'to' */
620 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
622 depdi 1, 41,1, %r29 /* Form aliased virtual address 'from' */
624 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
625 extrw,u %r23, 24,25, %r23 /* convert phys addr to tlb insert format */
626 depw %r24, 31,22, %r28 /* Form aliased virtual address 'to' */
627 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
629 depwi 1, 9,1, %r29 /* Form aliased virtual address 'from' */
632 /* Purge any old translations */
638 tlb_lock %r20,%r21,%r22
641 tlb_unlock %r20,%r21,%r22
645 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
646 * Unroll the loop by hand and arrange insn appropriately.
647 * GCC probably can do this just as well.
651 ldi (PAGE_SIZE / 128), %r1
695 /* conditional branches nullify on forward taken branch, and on
696 * non-taken backward branch. Note that .+4 is a backwards branch.
697 * The ldd should only get executed if the branch is taken.
699 addib,COND(>),n -1, %r1, 1b /* bundle 10 */
700 ldd 0(%r29), %r19 /* start next loads */
703 ldi (PAGE_SIZE / 64), %r1
706 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
707 * bundles (very restricted rules for bundling). It probably
708 * does OK on PCXU and better, but we could do better with
709 * ldd/std instructions. Note that until (if) we start saving
710 * the full 64 bit register values on interrupt, we can't
711 * use ldd/std on a 32 bit kernel.
748 addib,COND(>) -1, %r1,1b
757 ENDPROC_CFI(copy_user_page_asm)
759 ENTRY_CFI(clear_user_page_asm)
766 ldil L%(TMPALIAS_MAP_START), %r28
768 #if (TMPALIAS_MAP_START >= 0x80000000)
769 depdi 0, 31,32, %r28 /* clear any sign extension */
771 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
772 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
773 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
775 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
776 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
777 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
780 /* Purge any old translation */
785 tlb_lock %r20,%r21,%r22
787 tlb_unlock %r20,%r21,%r22
791 ldi (PAGE_SIZE / 128), %r1
793 /* PREFETCH (Write) has not (yet) been proven to help here */
794 /* #define PREFETCHW_OP ldd 256(%0), %r0 */
812 addib,COND(>) -1, %r1, 1b
815 #else /* ! CONFIG_64BIT */
816 ldi (PAGE_SIZE / 64), %r1
834 addib,COND(>) -1, %r1, 1b
836 #endif /* CONFIG_64BIT */
843 ENDPROC_CFI(clear_user_page_asm)
845 ENTRY_CFI(flush_dcache_page_asm)
850 ldil L%(TMPALIAS_MAP_START), %r28
852 #if (TMPALIAS_MAP_START >= 0x80000000)
853 depdi 0, 31,32, %r28 /* clear any sign extension */
855 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
856 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
857 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
859 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
860 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
861 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
864 /* Purge any old translation */
869 tlb_lock %r20,%r21,%r22
871 tlb_unlock %r20,%r21,%r22
874 ldil L%dcache_stride, %r1
875 ldw R%dcache_stride(%r1), r31
878 depdi,z 1, 63-PAGE_SHIFT,1, %r25
880 depwi,z 1, 31-PAGE_SHIFT,1, %r25
901 cmpb,COND(<<) %r28, %r25,1b
910 ENDPROC_CFI(flush_dcache_page_asm)
912 ENTRY_CFI(flush_icache_page_asm)
917 ldil L%(TMPALIAS_MAP_START), %r28
919 #if (TMPALIAS_MAP_START >= 0x80000000)
920 depdi 0, 31,32, %r28 /* clear any sign extension */
922 convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
923 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
924 depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
926 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
927 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
928 depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
931 /* Purge any old translation. Note that the FIC instruction
932 * may use either the instruction or data TLB. Given that we
933 * have a flat address space, it's not clear which TLB will be
934 * used. So, we purge both entries. */
938 pitlb,l %r0(%sr4,%r28)
940 tlb_lock %r20,%r21,%r22
943 tlb_unlock %r20,%r21,%r22
946 ldil L%icache_stride, %r1
947 ldw R%icache_stride(%r1), %r31
950 depdi,z 1, 63-PAGE_SHIFT,1, %r25
952 depwi,z 1, 31-PAGE_SHIFT,1, %r25
958 /* fic only has the type 26 form on PA1.1, requiring an
959 * explicit space specification, so use %sr4 */
960 1: fic,m %r31(%sr4,%r28)
961 fic,m %r31(%sr4,%r28)
962 fic,m %r31(%sr4,%r28)
963 fic,m %r31(%sr4,%r28)
964 fic,m %r31(%sr4,%r28)
965 fic,m %r31(%sr4,%r28)
966 fic,m %r31(%sr4,%r28)
967 fic,m %r31(%sr4,%r28)
968 fic,m %r31(%sr4,%r28)
969 fic,m %r31(%sr4,%r28)
970 fic,m %r31(%sr4,%r28)
971 fic,m %r31(%sr4,%r28)
972 fic,m %r31(%sr4,%r28)
973 fic,m %r31(%sr4,%r28)
974 fic,m %r31(%sr4,%r28)
975 cmpb,COND(<<) %r28, %r25,1b
976 fic,m %r31(%sr4,%r28)
984 ENDPROC_CFI(flush_icache_page_asm)
986 ENTRY_CFI(flush_kernel_dcache_page_asm)
991 ldil L%dcache_stride, %r1
992 ldw R%dcache_stride(%r1), %r23
995 depdi,z 1, 63-PAGE_SHIFT,1, %r25
997 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1000 sub %r25, %r23, %r25
1018 cmpb,COND(<<) %r26, %r25,1b
1027 ENDPROC_CFI(flush_kernel_dcache_page_asm)
1029 ENTRY_CFI(purge_kernel_dcache_page_asm)
1034 ldil L%dcache_stride, %r1
1035 ldw R%dcache_stride(%r1), %r23
1038 depdi,z 1, 63-PAGE_SHIFT,1, %r25
1040 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1042 add %r26, %r25, %r25
1043 sub %r25, %r23, %r25
1060 cmpb,COND(<<) %r26, %r25, 1b
1069 ENDPROC_CFI(purge_kernel_dcache_page_asm)
1071 ENTRY_CFI(flush_user_dcache_range_asm)
1076 ldil L%dcache_stride, %r1
1077 ldw R%dcache_stride(%r1), %r23
1079 ANDCM %r26, %r21, %r26
1081 1: cmpb,COND(<<),n %r26, %r25, 1b
1082 fdc,m %r23(%sr3, %r26)
1090 ENDPROC_CFI(flush_user_dcache_range_asm)
1092 ENTRY_CFI(flush_kernel_dcache_range_asm)
1097 ldil L%dcache_stride, %r1
1098 ldw R%dcache_stride(%r1), %r23
1100 ANDCM %r26, %r21, %r26
1102 1: cmpb,COND(<<),n %r26, %r25,1b
1112 ENDPROC_CFI(flush_kernel_dcache_range_asm)
1114 ENTRY_CFI(purge_kernel_dcache_range_asm)
1119 ldil L%dcache_stride, %r1
1120 ldw R%dcache_stride(%r1), %r23
1122 ANDCM %r26, %r21, %r26
1124 1: cmpb,COND(<<),n %r26, %r25,1b
1134 ENDPROC_CFI(purge_kernel_dcache_range_asm)
1136 ENTRY_CFI(flush_user_icache_range_asm)
1141 ldil L%icache_stride, %r1
1142 ldw R%icache_stride(%r1), %r23
1144 ANDCM %r26, %r21, %r26
1146 1: cmpb,COND(<<),n %r26, %r25,1b
1147 fic,m %r23(%sr3, %r26)
1155 ENDPROC_CFI(flush_user_icache_range_asm)
1157 ENTRY_CFI(flush_kernel_icache_page)
1162 ldil L%icache_stride, %r1
1163 ldw R%icache_stride(%r1), %r23
1166 depdi,z 1, 63-PAGE_SHIFT,1, %r25
1168 depwi,z 1, 31-PAGE_SHIFT,1, %r25
1170 add %r26, %r25, %r25
1171 sub %r25, %r23, %r25
1174 1: fic,m %r23(%sr4, %r26)
1175 fic,m %r23(%sr4, %r26)
1176 fic,m %r23(%sr4, %r26)
1177 fic,m %r23(%sr4, %r26)
1178 fic,m %r23(%sr4, %r26)
1179 fic,m %r23(%sr4, %r26)
1180 fic,m %r23(%sr4, %r26)
1181 fic,m %r23(%sr4, %r26)
1182 fic,m %r23(%sr4, %r26)
1183 fic,m %r23(%sr4, %r26)
1184 fic,m %r23(%sr4, %r26)
1185 fic,m %r23(%sr4, %r26)
1186 fic,m %r23(%sr4, %r26)
1187 fic,m %r23(%sr4, %r26)
1188 fic,m %r23(%sr4, %r26)
1189 cmpb,COND(<<) %r26, %r25, 1b
1190 fic,m %r23(%sr4, %r26)
1198 ENDPROC_CFI(flush_kernel_icache_page)
1200 ENTRY_CFI(flush_kernel_icache_range_asm)
1205 ldil L%icache_stride, %r1
1206 ldw R%icache_stride(%r1), %r23
1208 ANDCM %r26, %r21, %r26
1210 1: cmpb,COND(<<),n %r26, %r25, 1b
1211 fic,m %r23(%sr4, %r26)
1218 ENDPROC_CFI(flush_kernel_icache_range_asm)
1220 /* align should cover use of rfi in disable_sr_hashing_asm and
1224 ENTRY_CFI(disable_sr_hashing_asm)
1230 * Switch to real mode
1241 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1242 mtctl %r0, %cr17 /* Clear IIASQ tail */
1243 mtctl %r0, %cr17 /* Clear IIASQ head */
1244 mtctl %r1, %cr18 /* IIAOQ head */
1246 mtctl %r1, %cr18 /* IIAOQ tail */
1247 load32 REAL_MODE_PSW, %r1
1252 1: cmpib,=,n SRHASH_PCXST, %r26,srdis_pcxs
1253 cmpib,=,n SRHASH_PCXL, %r26,srdis_pcxl
1254 cmpib,=,n SRHASH_PA20, %r26,srdis_pa20
1259 /* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1261 .word 0x141c1a00 /* mfdiag %dr0, %r28 */
1262 .word 0x141c1a00 /* must issue twice */
1263 depwi 0,18,1, %r28 /* Clear DHE (dcache hash enable) */
1264 depwi 0,20,1, %r28 /* Clear IHE (icache hash enable) */
1265 .word 0x141c1600 /* mtdiag %r28, %dr0 */
1266 .word 0x141c1600 /* must issue twice */
1271 /* Disable Space Register Hashing for PCXL */
1273 .word 0x141c0600 /* mfdiag %dr0, %r28 */
1274 depwi 0,28,2, %r28 /* Clear DHASH_EN & IHASH_EN */
1275 .word 0x141c0240 /* mtdiag %r28, %dr0 */
1280 /* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1282 .word 0x144008bc /* mfdiag %dr2, %r28 */
1283 depdi 0, 54,1, %r28 /* clear DIAG_SPHASH_ENAB (bit 54) */
1284 .word 0x145c1840 /* mtdiag %r28, %dr2 */
1288 /* Switch back to virtual mode */
1289 rsm PSW_SM_I, %r0 /* prep to load iia queue */
1297 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1298 mtctl %r0, %cr17 /* Clear IIASQ tail */
1299 mtctl %r0, %cr17 /* Clear IIASQ head */
1300 mtctl %r1, %cr18 /* IIAOQ head */
1302 mtctl %r1, %cr18 /* IIAOQ tail */
1303 load32 KERNEL_PSW, %r1
1313 ENDPROC_CFI(disable_sr_hashing_asm)