GNU Linux-libre 4.4.292-gnu1
[releases.git] / arch / ia64 / kernel / fsys.S
1 /*
2  * This file contains the light-weight system call handlers (fsyscall-handlers).
3  *
4  * Copyright (C) 2003 Hewlett-Packard Co
5  *      David Mosberger-Tang <davidm@hpl.hp.com>
6  *
7  * 25-Sep-03 davidm     Implement fsys_rt_sigprocmask().
8  * 18-Feb-03 louisk     Implement fsys_gettimeofday().
9  * 28-Feb-03 davidm     Fixed several bugs in fsys_gettimeofday().  Tuned it some more,
10  *                      probably broke it along the way... ;-)
11  * 13-Jul-04 clameter   Implement fsys_clock_gettime and revise fsys_gettimeofday to make
12  *                      it capable of using memory based clocks without falling back to C code.
13  * 08-Feb-07 Fenghua Yu Implement fsys_getcpu.
14  *
15  */
16
17 #include <asm/asmmacro.h>
18 #include <asm/errno.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/percpu.h>
21 #include <asm/thread_info.h>
22 #include <asm/sal.h>
23 #include <asm/signal.h>
24 #include <asm/unistd.h>
25
26 #include "entry.h"
27 #include <asm/native/inst.h>
28
29 /*
30  * See Documentation/ia64/fsys.txt for details on fsyscalls.
31  *
32  * On entry to an fsyscall handler:
33  *   r10        = 0 (i.e., defaults to "successful syscall return")
34  *   r11        = saved ar.pfs (a user-level value)
35  *   r15        = system call number
36  *   r16        = "current" task pointer (in normal kernel-mode, this is in r13)
37  *   r32-r39    = system call arguments
38  *   b6         = return address (a user-level value)
39  *   ar.pfs     = previous frame-state (a user-level value)
40  *   PSR.be     = cleared to zero (i.e., little-endian byte order is in effect)
41  *   all other registers may contain values passed in from user-mode
42  *
43  * On return from an fsyscall handler:
44  *   r11        = saved ar.pfs (as passed into the fsyscall handler)
45  *   r15        = system call number (as passed into the fsyscall handler)
46  *   r32-r39    = system call arguments (as passed into the fsyscall handler)
47  *   b6         = return address (as passed into the fsyscall handler)
48  *   ar.pfs     = previous frame-state (as passed into the fsyscall handler)
49  */
50
51 ENTRY(fsys_ni_syscall)
52         .prologue
53         .altrp b6
54         .body
55         mov r8=ENOSYS
56         mov r10=-1
57         FSYS_RETURN
58 END(fsys_ni_syscall)
59
60 ENTRY(fsys_getpid)
61         .prologue
62         .altrp b6
63         .body
64         add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
65         ;;
66         ld8 r17=[r17]                           // r17 = current->group_leader
67         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
68         ;;
69         ld4 r9=[r9]
70         add r17=IA64_TASK_TGIDLINK_OFFSET,r17
71         ;;
72         and r9=TIF_ALLWORK_MASK,r9
73         ld8 r17=[r17]                           // r17 = current->group_leader->pids[PIDTYPE_PID].pid
74         ;;
75         add r8=IA64_PID_LEVEL_OFFSET,r17
76         ;;
77         ld4 r8=[r8]                             // r8 = pid->level
78         add r17=IA64_PID_UPID_OFFSET,r17        // r17 = &pid->numbers[0]
79         ;;
80         shl r8=r8,IA64_UPID_SHIFT
81         ;;
82         add r17=r17,r8                          // r17 = &pid->numbers[pid->level]
83         ;;
84         ld4 r8=[r17]                            // r8 = pid->numbers[pid->level].nr
85         ;;
86         mov r17=0
87         ;;
88         cmp.ne p8,p0=0,r9
89 (p8)    br.spnt.many fsys_fallback_syscall
90         FSYS_RETURN
91 END(fsys_getpid)
92
93 ENTRY(fsys_set_tid_address)
94         .prologue
95         .altrp b6
96         .body
97         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
98         add r17=IA64_TASK_TGIDLINK_OFFSET,r16
99         ;;
100         ld4 r9=[r9]
101         tnat.z p6,p7=r32                // check argument register for being NaT
102         ld8 r17=[r17]                           // r17 = current->pids[PIDTYPE_PID].pid
103         ;;
104         and r9=TIF_ALLWORK_MASK,r9
105         add r8=IA64_PID_LEVEL_OFFSET,r17
106         add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
107         ;;
108         ld4 r8=[r8]                             // r8 = pid->level
109         add r17=IA64_PID_UPID_OFFSET,r17        // r17 = &pid->numbers[0]
110         ;;
111         shl r8=r8,IA64_UPID_SHIFT
112         ;;
113         add r17=r17,r8                          // r17 = &pid->numbers[pid->level]
114         ;;
115         ld4 r8=[r17]                            // r8 = pid->numbers[pid->level].nr
116         ;;
117         cmp.ne p8,p0=0,r9
118         mov r17=-1
119         ;;
120 (p6)    st8 [r18]=r32
121 (p7)    st8 [r18]=r17
122 (p8)    br.spnt.many fsys_fallback_syscall
123         ;;
124         mov r17=0                       // i must not leak kernel bits...
125         mov r18=0                       // i must not leak kernel bits...
126         FSYS_RETURN
127 END(fsys_set_tid_address)
128
129 #if IA64_GTOD_SEQ_OFFSET !=0
130 #error fsys_gettimeofday incompatible with changes to struct fsyscall_gtod_data_t
131 #endif
132 #if IA64_ITC_JITTER_OFFSET !=0
133 #error fsys_gettimeofday incompatible with changes to struct itc_jitter_data_t
134 #endif
135 #define CLOCK_REALTIME 0
136 #define CLOCK_MONOTONIC 1
137 #define CLOCK_DIVIDE_BY_1000 0x4000
138 #define CLOCK_ADD_MONOTONIC 0x8000
139
140 ENTRY(fsys_gettimeofday)
141         .prologue
142         .altrp b6
143         .body
144         mov r31 = r32
145         tnat.nz p6,p0 = r33             // guard against NaT argument
146 (p6)    br.cond.spnt.few .fail_einval
147         mov r30 = CLOCK_DIVIDE_BY_1000
148         ;;
149 .gettime:
150         // Register map
151         // Incoming r31 = pointer to address where to place result
152         //          r30 = flags determining how time is processed
153         // r2,r3 = temp r4-r7 preserved
154         // r8 = result nanoseconds
155         // r9 = result seconds
156         // r10 = temporary storage for clock difference
157         // r11 = preserved: saved ar.pfs
158         // r12 = preserved: memory stack
159         // r13 = preserved: thread pointer
160         // r14 = address of mask / mask value
161         // r15 = preserved: system call number
162         // r16 = preserved: current task pointer
163         // r17 = (not used)
164         // r18 = (not used)
165         // r19 = address of itc_lastcycle
166         // r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
167         // r21 = address of mmio_ptr
168         // r22 = address of wall_time or monotonic_time
169         // r23 = address of shift / value
170         // r24 = address mult factor / cycle_last value
171         // r25 = itc_lastcycle value
172         // r26 = address clocksource cycle_last
173         // r27 = (not used)
174         // r28 = sequence number at the beginning of critcal section
175         // r29 = address of itc_jitter
176         // r30 = time processing flags / memory address
177         // r31 = pointer to result
178         // Predicates
179         // p6,p7 short term use
180         // p8 = timesource ar.itc
181         // p9 = timesource mmio64
182         // p10 = timesource mmio32 - not used
183         // p11 = timesource not to be handled by asm code
184         // p12 = memory time source ( = p9 | p10) - not used
185         // p13 = do cmpxchg with itc_lastcycle
186         // p14 = Divide by 1000
187         // p15 = Add monotonic
188         //
189         // Note that instructions are optimized for McKinley. McKinley can
190         // process two bundles simultaneously and therefore we continuously
191         // try to feed the CPU two bundles and then a stop.
192
193         add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
194         tnat.nz p6,p0 = r31             // guard against Nat argument
195 (p6)    br.cond.spnt.few .fail_einval
196         movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
197         ;;
198         ld4 r2 = [r2]                   // process work pending flags
199         movl r29 = itc_jitter_data      // itc_jitter
200         add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20        // wall_time
201         add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
202         mov pr = r30,0xc000     // Set predicates according to function
203         ;;
204         and r2 = TIF_ALLWORK_MASK,r2
205         add r19 = IA64_ITC_LASTCYCLE_OFFSET,r29
206 (p15)   add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20        // monotonic_time
207         ;;
208         add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20     // clksrc_cycle_last
209         cmp.ne p6, p0 = 0, r2   // Fallback if work is scheduled
210 (p6)    br.cond.spnt.many fsys_fallback_syscall
211         ;;
212         // Begin critical section
213 .time_redo:
214         ld4.acq r28 = [r20]     // gtod_lock.sequence, Must take first
215         ;;
216         and r28 = ~1,r28        // And make sequence even to force retry if odd
217         ;;
218         ld8 r30 = [r21]         // clocksource->mmio_ptr
219         add r24 = IA64_CLKSRC_MULT_OFFSET,r20
220         ld4 r2 = [r29]          // itc_jitter value
221         add r23 = IA64_CLKSRC_SHIFT_OFFSET,r20
222         add r14 = IA64_CLKSRC_MASK_OFFSET,r20
223         ;;
224         ld4 r3 = [r24]          // clocksource mult value
225         ld8 r14 = [r14]         // clocksource mask value
226         cmp.eq p8,p9 = 0,r30    // use cpu timer if no mmio_ptr
227         ;;
228         setf.sig f7 = r3        // Setup for mult scaling of counter
229 (p8)    cmp.ne p13,p0 = r2,r0   // need itc_jitter compensation, set p13
230         ld4 r23 = [r23]         // clocksource shift value
231         ld8 r24 = [r26]         // get clksrc_cycle_last value
232 (p9)    cmp.eq p13,p0 = 0,r30   // if mmio_ptr, clear p13 jitter control
233         ;;
234         .pred.rel.mutex p8,p9
235         MOV_FROM_ITC(p8, p6, r2, r10)   // CPU_TIMER. 36 clocks latency!!!
236 (p9)    ld8 r2 = [r30]          // MMIO_TIMER. Could also have latency issues..
237 (p13)   ld8 r25 = [r19]         // get itc_lastcycle value
238         ld8 r9 = [r22],IA64_TIMESPEC_TV_NSEC_OFFSET     // tv_sec
239         ;;
240         ld8 r8 = [r22],-IA64_TIMESPEC_TV_NSEC_OFFSET    // tv_nsec
241 (p13)   sub r3 = r25,r2         // Diff needed before comparison (thanks davidm)
242         ;;
243 (p13)   cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
244         sub r10 = r2,r24        // current_cycle - last_cycle
245         ;;
246 (p6)    sub r10 = r25,r24       // time we got was less than last_cycle
247 (p7)    mov ar.ccv = r25        // more than last_cycle. Prep for cmpxchg
248         ;;
249 (p7)    cmpxchg8.rel r3 = [r19],r2,ar.ccv
250         ;;
251 (p7)    cmp.ne p7,p0 = r25,r3   // if cmpxchg not successful
252         ;;
253 (p7)    sub r10 = r3,r24        // then use new last_cycle instead
254         ;;
255         and r10 = r10,r14       // Apply mask
256         ;;
257         setf.sig f8 = r10
258         nop.i 123
259         ;;
260         // fault check takes 5 cycles and we have spare time
261 EX(.fail_efault, probe.w.fault r31, 3)
262         xmpy.l f8 = f8,f7       // nsec_per_cyc*(counter-last_counter)
263         ;;
264         getf.sig r2 = f8
265         mf
266         ;;
267         ld4 r10 = [r20]         // gtod_lock.sequence
268         shr.u r2 = r2,r23       // shift by factor
269         ;;
270         add r8 = r8,r2          // Add xtime.nsecs
271         cmp4.ne p7,p0 = r28,r10
272 (p7)    br.cond.dpnt.few .time_redo     // sequence number changed, redo
273         // End critical section.
274         // Now r8=tv->tv_nsec and r9=tv->tv_sec
275         mov r10 = r0
276         movl r2 = 1000000000
277         add r23 = IA64_TIMESPEC_TV_NSEC_OFFSET, r31
278 (p14)   movl r3 = 2361183241434822607   // Prep for / 1000 hack
279         ;;
280 .time_normalize:
281         mov r21 = r8
282         cmp.ge p6,p0 = r8,r2
283 (p14)   shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
284         ;;
285 (p14)   setf.sig f8 = r20
286 (p6)    sub r8 = r8,r2
287 (p6)    add r9 = 1,r9           // two nops before the branch.
288 (p14)   setf.sig f7 = r3        // Chances for repeats are 1 in 10000 for gettod
289 (p6)    br.cond.dpnt.few .time_normalize
290         ;;
291         // Divided by 8 though shift. Now divide by 125
292         // The compiler was able to do that with a multiply
293         // and a shift and we do the same
294 EX(.fail_efault, probe.w.fault r23, 3)  // This also costs 5 cycles
295 (p14)   xmpy.hu f8 = f8, f7             // xmpy has 5 cycles latency so use it
296         ;;
297 (p14)   getf.sig r2 = f8
298         ;;
299         mov r8 = r0
300 (p14)   shr.u r21 = r2, 4
301         ;;
302 EX(.fail_efault, st8 [r31] = r9)
303 EX(.fail_efault, st8 [r23] = r21)
304         FSYS_RETURN
305 .fail_einval:
306         mov r8 = EINVAL
307         mov r10 = -1
308         FSYS_RETURN
309 .fail_efault:
310         mov r8 = EFAULT
311         mov r10 = -1
312         FSYS_RETURN
313 END(fsys_gettimeofday)
314
315 ENTRY(fsys_clock_gettime)
316         .prologue
317         .altrp b6
318         .body
319         cmp4.ltu p6, p0 = CLOCK_MONOTONIC, r32
320         // Fallback if this is not CLOCK_REALTIME or CLOCK_MONOTONIC
321 (p6)    br.spnt.few fsys_fallback_syscall
322         mov r31 = r33
323         shl r30 = r32,15
324         br.many .gettime
325 END(fsys_clock_gettime)
326
327 /*
328  * fsys_getcpu doesn't use the third parameter in this implementation. It reads
329  * current_thread_info()->cpu and corresponding node in cpu_to_node_map.
330  */
331 ENTRY(fsys_getcpu)
332         .prologue
333         .altrp b6
334         .body
335         ;;
336         add r2=TI_FLAGS+IA64_TASK_SIZE,r16
337         tnat.nz p6,p0 = r32                     // guard against NaT argument
338         add r3=TI_CPU+IA64_TASK_SIZE,r16
339         ;;
340         ld4 r3=[r3]                             // M r3 = thread_info->cpu
341         ld4 r2=[r2]                             // M r2 = thread_info->flags
342 (p6)    br.cond.spnt.few .fail_einval           // B
343         ;;
344         tnat.nz p7,p0 = r33                     // I guard against NaT argument
345 (p7)    br.cond.spnt.few .fail_einval           // B
346         ;;
347         cmp.ne p6,p0=r32,r0
348         cmp.ne p7,p0=r33,r0
349         ;;
350 #ifdef CONFIG_NUMA
351         movl r17=cpu_to_node_map
352         ;;
353 EX(.fail_efault, (p6) probe.w.fault r32, 3)             // M This takes 5 cycles
354 EX(.fail_efault, (p7) probe.w.fault r33, 3)             // M This takes 5 cycles
355         shladd r18=r3,1,r17
356         ;;
357         ld2 r20=[r18]                           // r20 = cpu_to_node_map[cpu]
358         and r2 = TIF_ALLWORK_MASK,r2
359         ;;
360         cmp.ne p8,p0=0,r2
361 (p8)    br.spnt.many fsys_fallback_syscall
362         ;;
363         ;;
364 EX(.fail_efault, (p6) st4 [r32] = r3)
365 EX(.fail_efault, (p7) st2 [r33] = r20)
366         mov r8=0
367         ;;
368 #else
369 EX(.fail_efault, (p6) probe.w.fault r32, 3)             // M This takes 5 cycles
370 EX(.fail_efault, (p7) probe.w.fault r33, 3)             // M This takes 5 cycles
371         and r2 = TIF_ALLWORK_MASK,r2
372         ;;
373         cmp.ne p8,p0=0,r2
374 (p8)    br.spnt.many fsys_fallback_syscall
375         ;;
376 EX(.fail_efault, (p6) st4 [r32] = r3)
377 EX(.fail_efault, (p7) st2 [r33] = r0)
378         mov r8=0
379         ;;
380 #endif
381         FSYS_RETURN
382 END(fsys_getcpu)
383
384 ENTRY(fsys_fallback_syscall)
385         .prologue
386         .altrp b6
387         .body
388         /*
389          * We only get here from light-weight syscall handlers.  Thus, we already
390          * know that r15 contains a valid syscall number.  No need to re-check.
391          */
392         adds r17=-1024,r15
393         movl r14=sys_call_table
394         ;;
395         RSM_PSR_I(p0, r26, r27)
396         shladd r18=r17,3,r14
397         ;;
398         ld8 r18=[r18]                           // load normal (heavy-weight) syscall entry-point
399         MOV_FROM_PSR(p0, r29, r26)              // read psr (12 cyc load latency)
400         mov r27=ar.rsc
401         mov r21=ar.fpsr
402         mov r26=ar.pfs
403 END(fsys_fallback_syscall)
404         /* FALL THROUGH */
405 GLOBAL_ENTRY(fsys_bubble_down)
406         .prologue
407         .altrp b6
408         .body
409         /*
410          * We get here for syscalls that don't have a lightweight
411          * handler.  For those, we need to bubble down into the kernel
412          * and that requires setting up a minimal pt_regs structure,
413          * and initializing the CPU state more or less as if an
414          * interruption had occurred.  To make syscall-restarts work,
415          * we setup pt_regs such that cr_iip points to the second
416          * instruction in syscall_via_break.  Decrementing the IP
417          * hence will restart the syscall via break and not
418          * decrementing IP will return us to the caller, as usual.
419          * Note that we preserve the value of psr.pp rather than
420          * initializing it from dcr.pp.  This makes it possible to
421          * distinguish fsyscall execution from other privileged
422          * execution.
423          *
424          * On entry:
425          *      - normal fsyscall handler register usage, except
426          *        that we also have:
427          *      - r18: address of syscall entry point
428          *      - r21: ar.fpsr
429          *      - r26: ar.pfs
430          *      - r27: ar.rsc
431          *      - r29: psr
432          *
433          * We used to clear some PSR bits here but that requires slow
434          * serialization.  Fortuntely, that isn't really necessary.
435          * The rationale is as follows: we used to clear bits
436          * ~PSR_PRESERVED_BITS in PSR.L.  Since
437          * PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
438          * ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
439          * However,
440          *
441          * PSR.BE : already is turned off in __kernel_syscall_via_epc()
442          * PSR.AC : don't care (kernel normally turns PSR.AC on)
443          * PSR.I  : already turned off by the time fsys_bubble_down gets
444          *          invoked
445          * PSR.DFL: always 0 (kernel never turns it on)
446          * PSR.DFH: don't care --- kernel never touches f32-f127 on its own
447          *          initiative
448          * PSR.DI : always 0 (kernel never turns it on)
449          * PSR.SI : always 0 (kernel never turns it on)
450          * PSR.DB : don't care --- kernel never enables kernel-level
451          *          breakpoints
452          * PSR.TB : must be 0 already; if it wasn't zero on entry to
453          *          __kernel_syscall_via_epc, the branch to fsys_bubble_down
454          *          will trigger a taken branch; the taken-trap-handler then
455          *          converts the syscall into a break-based system-call.
456          */
457         /*
458          * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
459          * The rest we have to synthesize.
460          */
461 #       define PSR_ONE_BITS             ((3 << IA64_PSR_CPL0_BIT)       \
462                                          | (0x1 << IA64_PSR_RI_BIT)     \
463                                          | IA64_PSR_BN | IA64_PSR_I)
464
465         invala                                  // M0|1
466         movl r14=ia64_ret_from_syscall          // X
467
468         nop.m 0
469         movl r28=__kernel_syscall_via_break     // X    create cr.iip
470         ;;
471
472         mov r2=r16                              // A    get task addr to addl-addressable register
473         adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
474         mov r31=pr                              // I0   save pr (2 cyc)
475         ;;
476         st1 [r16]=r0                            // M2|3 clear current->thread.on_ustack flag
477         addl r22=IA64_RBS_OFFSET,r2             // A    compute base of RBS
478         add r3=TI_FLAGS+IA64_TASK_SIZE,r2       // A
479         ;;
480         ld4 r3=[r3]                             // M0|1 r3 = current_thread_info()->flags
481         lfetch.fault.excl.nt1 [r22]             // M0|1 prefetch register backing-store
482         nop.i 0
483         ;;
484         mov ar.rsc=0                            // M2   set enforced lazy mode, pl 0, LE, loadrs=0
485 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
486         MOV_FROM_ITC(p0, p6, r30, r23)          // M    get cycle for accounting
487 #else
488         nop.m 0
489 #endif
490         nop.i 0
491         ;;
492         mov r23=ar.bspstore                     // M2 (12 cyc) save ar.bspstore
493         mov.m r24=ar.rnat                       // M2 (5 cyc) read ar.rnat (dual-issues!)
494         nop.i 0
495         ;;
496         mov ar.bspstore=r22                     // M2 (6 cyc) switch to kernel RBS
497         movl r8=PSR_ONE_BITS                    // X
498         ;;
499         mov r25=ar.unat                         // M2 (5 cyc) save ar.unat
500         mov r19=b6                              // I0   save b6 (2 cyc)
501         mov r20=r1                              // A    save caller's gp in r20
502         ;;
503         or r29=r8,r29                           // A    construct cr.ipsr value to save
504         mov b6=r18                              // I0   copy syscall entry-point to b6 (7 cyc)
505         addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
506
507         mov r18=ar.bsp                          // M2   save (kernel) ar.bsp (12 cyc)
508         cmp.ne pKStk,pUStk=r0,r0                // A    set pKStk <- 0, pUStk <- 1
509         br.call.sptk.many b7=ia64_syscall_setup // B
510         ;;
511 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
512         // mov.m r30=ar.itc is called in advance
513         add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
514         add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
515         ;;
516         ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP   // time at last check in kernel
517         ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE   // time at leave kernel
518         ;;
519         ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME   // cumulated stime
520         ld8 r21=[r17]                           // cumulated utime
521         sub r22=r19,r18                         // stime before leave kernel
522         ;;
523         st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP   // update stamp
524         sub r18=r30,r19                         // elapsed time in user mode
525         ;;
526         add r20=r20,r22                         // sum stime
527         add r21=r21,r18                         // sum utime
528         ;;
529         st8 [r16]=r20                           // update stime
530         st8 [r17]=r21                           // update utime
531         ;;
532 #endif
533         mov ar.rsc=0x3                          // M2   set eager mode, pl 0, LE, loadrs=0
534         mov rp=r14                              // I0   set the real return addr
535         and r3=_TIF_SYSCALL_TRACEAUDIT,r3       // A
536         ;;
537         SSM_PSR_I(p0, p6, r22)                  // M2   we're on kernel stacks now, reenable irqs
538         cmp.eq p8,p0=r3,r0                      // A
539 (p10)   br.cond.spnt.many ia64_ret_from_syscall // B    return if bad call-frame or r15 is a NaT
540
541         nop.m 0
542 (p8)    br.call.sptk.many b6=b6                 // B    (ignore return address)
543         br.cond.spnt ia64_trace_syscall         // B
544 END(fsys_bubble_down)
545
546         .rodata
547         .align 8
548         .globl fsyscall_table
549
550         data8 fsys_bubble_down
551 fsyscall_table:
552         data8 fsys_ni_syscall
553         data8 0                         // exit                 // 1025
554         data8 0                         // read
555         data8 0                         // write
556         data8 0                         // open
557         data8 0                         // close
558         data8 0                         // creat                // 1030
559         data8 0                         // link
560         data8 0                         // unlink
561         data8 0                         // execve
562         data8 0                         // chdir
563         data8 0                         // fchdir               // 1035
564         data8 0                         // utimes
565         data8 0                         // mknod
566         data8 0                         // chmod
567         data8 0                         // chown
568         data8 0                         // lseek                // 1040
569         data8 fsys_getpid               // getpid
570         data8 0                         // getppid
571         data8 0                         // mount
572         data8 0                         // umount
573         data8 0                         // setuid               // 1045
574         data8 0                         // getuid
575         data8 0                         // geteuid
576         data8 0                         // ptrace
577         data8 0                         // access
578         data8 0                         // sync                 // 1050
579         data8 0                         // fsync
580         data8 0                         // fdatasync
581         data8 0                         // kill
582         data8 0                         // rename
583         data8 0                         // mkdir                // 1055
584         data8 0                         // rmdir
585         data8 0                         // dup
586         data8 0                         // pipe
587         data8 0                         // times
588         data8 0                         // brk                  // 1060
589         data8 0                         // setgid
590         data8 0                         // getgid
591         data8 0                         // getegid
592         data8 0                         // acct
593         data8 0                         // ioctl                // 1065
594         data8 0                         // fcntl
595         data8 0                         // umask
596         data8 0                         // chroot
597         data8 0                         // ustat
598         data8 0                         // dup2                 // 1070
599         data8 0                         // setreuid
600         data8 0                         // setregid
601         data8 0                         // getresuid
602         data8 0                         // setresuid
603         data8 0                         // getresgid            // 1075
604         data8 0                         // setresgid
605         data8 0                         // getgroups
606         data8 0                         // setgroups
607         data8 0                         // getpgid
608         data8 0                         // setpgid              // 1080
609         data8 0                         // setsid
610         data8 0                         // getsid
611         data8 0                         // sethostname
612         data8 0                         // setrlimit
613         data8 0                         // getrlimit            // 1085
614         data8 0                         // getrusage
615         data8 fsys_gettimeofday         // gettimeofday
616         data8 0                         // settimeofday
617         data8 0                         // select
618         data8 0                         // poll                 // 1090
619         data8 0                         // symlink
620         data8 0                         // readlink
621         data8 0                         // uselib
622         data8 0                         // swapon
623         data8 0                         // swapoff              // 1095
624         data8 0                         // reboot
625         data8 0                         // truncate
626         data8 0                         // ftruncate
627         data8 0                         // fchmod
628         data8 0                         // fchown               // 1100
629         data8 0                         // getpriority
630         data8 0                         // setpriority
631         data8 0                         // statfs
632         data8 0                         // fstatfs
633         data8 0                         // gettid               // 1105
634         data8 0                         // semget
635         data8 0                         // semop
636         data8 0                         // semctl
637         data8 0                         // msgget
638         data8 0                         // msgsnd               // 1110
639         data8 0                         // msgrcv
640         data8 0                         // msgctl
641         data8 0                         // shmget
642         data8 0                         // shmat
643         data8 0                         // shmdt                // 1115
644         data8 0                         // shmctl
645         data8 0                         // syslog
646         data8 0                         // setitimer
647         data8 0                         // getitimer
648         data8 0                                                 // 1120
649         data8 0
650         data8 0
651         data8 0                         // vhangup
652         data8 0                         // lchown
653         data8 0                         // remap_file_pages     // 1125
654         data8 0                         // wait4
655         data8 0                         // sysinfo
656         data8 0                         // clone
657         data8 0                         // setdomainname
658         data8 0                         // newuname             // 1130
659         data8 0                         // adjtimex
660         data8 0
661         data8 0                         // init_module
662         data8 0                         // delete_module
663         data8 0                                                 // 1135
664         data8 0
665         data8 0                         // quotactl
666         data8 0                         // bdflush
667         data8 0                         // sysfs
668         data8 0                         // personality          // 1140
669         data8 0                         // afs_syscall
670         data8 0                         // setfsuid
671         data8 0                         // setfsgid
672         data8 0                         // getdents
673         data8 0                         // flock                // 1145
674         data8 0                         // readv
675         data8 0                         // writev
676         data8 0                         // pread64
677         data8 0                         // pwrite64
678         data8 0                         // sysctl               // 1150
679         data8 0                         // mmap
680         data8 0                         // munmap
681         data8 0                         // mlock
682         data8 0                         // mlockall
683         data8 0                         // mprotect             // 1155
684         data8 0                         // mremap
685         data8 0                         // msync
686         data8 0                         // munlock
687         data8 0                         // munlockall
688         data8 0                         // sched_getparam       // 1160
689         data8 0                         // sched_setparam
690         data8 0                         // sched_getscheduler
691         data8 0                         // sched_setscheduler
692         data8 0                         // sched_yield
693         data8 0                         // sched_get_priority_max       // 1165
694         data8 0                         // sched_get_priority_min
695         data8 0                         // sched_rr_get_interval
696         data8 0                         // nanosleep
697         data8 0                         // nfsservctl
698         data8 0                         // prctl                // 1170
699         data8 0                         // getpagesize
700         data8 0                         // mmap2
701         data8 0                         // pciconfig_read
702         data8 0                         // pciconfig_write
703         data8 0                         // perfmonctl           // 1175
704         data8 0                         // sigaltstack
705         data8 0                         // rt_sigaction
706         data8 0                         // rt_sigpending
707         data8 0                         // rt_sigprocmask
708         data8 0                         // rt_sigqueueinfo      // 1180
709         data8 0                         // rt_sigreturn
710         data8 0                         // rt_sigsuspend
711         data8 0                         // rt_sigtimedwait
712         data8 0                         // getcwd
713         data8 0                         // capget               // 1185
714         data8 0                         // capset
715         data8 0                         // sendfile
716         data8 0
717         data8 0
718         data8 0                         // socket               // 1190
719         data8 0                         // bind
720         data8 0                         // connect
721         data8 0                         // listen
722         data8 0                         // accept
723         data8 0                         // getsockname          // 1195
724         data8 0                         // getpeername
725         data8 0                         // socketpair
726         data8 0                         // send
727         data8 0                         // sendto
728         data8 0                         // recv                 // 1200
729         data8 0                         // recvfrom
730         data8 0                         // shutdown
731         data8 0                         // setsockopt
732         data8 0                         // getsockopt
733         data8 0                         // sendmsg              // 1205
734         data8 0                         // recvmsg
735         data8 0                         // pivot_root
736         data8 0                         // mincore
737         data8 0                         // madvise
738         data8 0                         // newstat              // 1210
739         data8 0                         // newlstat
740         data8 0                         // newfstat
741         data8 0                         // clone2
742         data8 0                         // getdents64
743         data8 0                         // getunwind            // 1215
744         data8 0                         // readahead
745         data8 0                         // setxattr
746         data8 0                         // lsetxattr
747         data8 0                         // fsetxattr
748         data8 0                         // getxattr             // 1220
749         data8 0                         // lgetxattr
750         data8 0                         // fgetxattr
751         data8 0                         // listxattr
752         data8 0                         // llistxattr
753         data8 0                         // flistxattr           // 1225
754         data8 0                         // removexattr
755         data8 0                         // lremovexattr
756         data8 0                         // fremovexattr
757         data8 0                         // tkill
758         data8 0                         // futex                // 1230
759         data8 0                         // sched_setaffinity
760         data8 0                         // sched_getaffinity
761         data8 fsys_set_tid_address      // set_tid_address
762         data8 0                         // fadvise64_64
763         data8 0                         // tgkill               // 1235
764         data8 0                         // exit_group
765         data8 0                         // lookup_dcookie
766         data8 0                         // io_setup
767         data8 0                         // io_destroy
768         data8 0                         // io_getevents         // 1240
769         data8 0                         // io_submit
770         data8 0                         // io_cancel
771         data8 0                         // epoll_create
772         data8 0                         // epoll_ctl
773         data8 0                         // epoll_wait           // 1245
774         data8 0                         // restart_syscall
775         data8 0                         // semtimedop
776         data8 0                         // timer_create
777         data8 0                         // timer_settime
778         data8 0                         // timer_gettime        // 1250
779         data8 0                         // timer_getoverrun
780         data8 0                         // timer_delete
781         data8 0                         // clock_settime
782         data8 fsys_clock_gettime        // clock_gettime
783         data8 0                         // clock_getres         // 1255
784         data8 0                         // clock_nanosleep
785         data8 0                         // fstatfs64
786         data8 0                         // statfs64
787         data8 0                         // mbind
788         data8 0                         // get_mempolicy        // 1260
789         data8 0                         // set_mempolicy
790         data8 0                         // mq_open
791         data8 0                         // mq_unlink
792         data8 0                         // mq_timedsend
793         data8 0                         // mq_timedreceive      // 1265
794         data8 0                         // mq_notify
795         data8 0                         // mq_getsetattr
796         data8 0                         // kexec_load
797         data8 0                         // vserver
798         data8 0                         // waitid               // 1270
799         data8 0                         // add_key
800         data8 0                         // request_key
801         data8 0                         // keyctl
802         data8 0                         // ioprio_set
803         data8 0                         // ioprio_get           // 1275
804         data8 0                         // move_pages
805         data8 0                         // inotify_init
806         data8 0                         // inotify_add_watch
807         data8 0                         // inotify_rm_watch
808         data8 0                         // migrate_pages        // 1280
809         data8 0                         // openat
810         data8 0                         // mkdirat
811         data8 0                         // mknodat
812         data8 0                         // fchownat
813         data8 0                         // futimesat            // 1285
814         data8 0                         // newfstatat
815         data8 0                         // unlinkat
816         data8 0                         // renameat
817         data8 0                         // linkat
818         data8 0                         // symlinkat            // 1290
819         data8 0                         // readlinkat
820         data8 0                         // fchmodat
821         data8 0                         // faccessat
822         data8 0
823         data8 0                                                 // 1295
824         data8 0                         // unshare
825         data8 0                         // splice
826         data8 0                         // set_robust_list
827         data8 0                         // get_robust_list
828         data8 0                         // sync_file_range      // 1300
829         data8 0                         // tee
830         data8 0                         // vmsplice
831         data8 0
832         data8 fsys_getcpu               // getcpu               // 1304
833
834         // fill in zeros for the remaining entries
835         .zero:
836         .space fsyscall_table + 8*NR_syscalls - .zero, 0