1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright IBM Corp. 1999, 2023
6 #include <linux/cpuhotplug.h>
7 #include <linux/sched/task.h>
8 #include <linux/errno.h>
9 #include <linux/init.h>
10 #include <linux/irq.h>
11 #include <asm/asm-extable.h>
12 #include <asm/pfault.h>
15 #define __SUBCODE_MASK 0x0600
16 #define __PF_RES_FIELD 0x8000000000000000UL
19 * 'pfault' pseudo page faults routines.
21 static int pfault_disable;
23 static int __init nopfault(char *str)
28 early_param("nopfault", nopfault);
41 static struct pfault_refbk pfault_init_refbk = {
47 .refselmk = 1UL << 48,
48 .refcmpmk = 1UL << 48,
49 .reserved = __PF_RES_FIELD
52 int __pfault_init(void)
58 diag_stat_inc(DIAG_STAT_X258);
60 " diag %[refbk],%[rc],0x258\n"
64 : [refbk] "a" (&pfault_init_refbk), "m" (pfault_init_refbk)
69 static struct pfault_refbk pfault_fini_refbk = {
76 void __pfault_fini(void)
80 diag_stat_inc(DIAG_STAT_X258);
82 " diag %[refbk],0,0x258\n"
86 : [refbk] "a" (&pfault_fini_refbk), "m" (pfault_fini_refbk)
90 static DEFINE_SPINLOCK(pfault_lock);
91 static LIST_HEAD(pfault_list);
93 #define PF_COMPLETE 0x0080
96 * The mechanism of our pfault code: if Linux is running as guest, runs a user
97 * space process and the user space process accesses a page that the host has
98 * paged out we get a pfault interrupt.
100 * This allows us, within the guest, to schedule a different process. Without
101 * this mechanism the host would have to suspend the whole virtual cpu until
102 * the page has been paged in.
104 * So when we get such an interrupt then we set the state of the current task
105 * to uninterruptible and also set the need_resched flag. Both happens within
106 * interrupt context(!). If we later on want to return to user space we
107 * recognize the need_resched flag and then call schedule(). It's not very
108 * obvious how this works...
110 * Of course we have a lot of additional fun with the completion interrupt (->
111 * host signals that a page of a process has been paged in and the process can
112 * continue to run). This interrupt can arrive on any cpu and, since we have
113 * virtual cpus, actually appear before the interrupt that signals that a page
116 static void pfault_interrupt(struct ext_code ext_code,
117 unsigned int param32, unsigned long param64)
119 struct task_struct *tsk;
124 * Get the external interruption subcode & pfault initial/completion
125 * signal bit. VM stores this in the 'cpu address' field associated
126 * with the external interrupt.
128 subcode = ext_code.subcode;
129 if ((subcode & 0xff00) != __SUBCODE_MASK)
131 inc_irq_stat(IRQEXT_PFL);
132 /* Get the token (= pid of the affected task). */
133 pid = param64 & LPP_PID_MASK;
135 tsk = find_task_by_pid_ns(pid, &init_pid_ns);
137 get_task_struct(tsk);
141 spin_lock(&pfault_lock);
142 if (subcode & PF_COMPLETE) {
143 /* signal bit is set -> a page has been swapped in by VM */
144 if (tsk->thread.pfault_wait == 1) {
146 * Initial interrupt was faster than the completion
147 * interrupt. pfault_wait is valid. Set pfault_wait
148 * back to zero and wake up the process. This can
149 * safely be done because the task is still sleeping
150 * and can't produce new pfaults.
152 tsk->thread.pfault_wait = 0;
153 list_del(&tsk->thread.list);
154 wake_up_process(tsk);
155 put_task_struct(tsk);
158 * Completion interrupt was faster than initial
159 * interrupt. Set pfault_wait to -1 so the initial
160 * interrupt doesn't put the task to sleep.
161 * If the task is not running, ignore the completion
162 * interrupt since it must be a leftover of a PFAULT
163 * CANCEL operation which didn't remove all pending
164 * completion interrupts.
166 if (task_is_running(tsk))
167 tsk->thread.pfault_wait = -1;
170 /* signal bit not set -> a real page is missing. */
171 if (WARN_ON_ONCE(tsk != current))
173 if (tsk->thread.pfault_wait == 1) {
174 /* Already on the list with a reference: put to sleep */
176 } else if (tsk->thread.pfault_wait == -1) {
178 * Completion interrupt was faster than the initial
179 * interrupt (pfault_wait == -1). Set pfault_wait
180 * back to zero and exit.
182 tsk->thread.pfault_wait = 0;
185 * Initial interrupt arrived before completion
186 * interrupt. Let the task sleep.
187 * An extra task reference is needed since a different
188 * cpu may set the task state to TASK_RUNNING again
189 * before the scheduler is reached.
191 get_task_struct(tsk);
192 tsk->thread.pfault_wait = 1;
193 list_add(&tsk->thread.list, &pfault_list);
196 * Since this must be a userspace fault, there
197 * is no kernel task state to trample. Rely on the
198 * return to userspace schedule() to block.
200 __set_current_state(TASK_UNINTERRUPTIBLE);
201 set_tsk_need_resched(tsk);
202 set_preempt_need_resched();
206 spin_unlock(&pfault_lock);
207 put_task_struct(tsk);
210 static int pfault_cpu_dead(unsigned int cpu)
212 struct thread_struct *thread, *next;
213 struct task_struct *tsk;
215 spin_lock_irq(&pfault_lock);
216 list_for_each_entry_safe(thread, next, &pfault_list, list) {
217 thread->pfault_wait = 0;
218 list_del(&thread->list);
219 tsk = container_of(thread, struct task_struct, thread);
220 wake_up_process(tsk);
221 put_task_struct(tsk);
223 spin_unlock_irq(&pfault_lock);
227 static int __init pfault_irq_init(void)
231 rc = register_external_irq(EXT_IRQ_CP_SERVICE, pfault_interrupt);
234 rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP;
237 irq_subclass_register(IRQ_SUBCLASS_SERVICE_SIGNAL);
238 cpuhp_setup_state_nocalls(CPUHP_S390_PFAULT_DEAD, "s390/pfault:dead",
239 NULL, pfault_cpu_dead);
243 unregister_external_irq(EXT_IRQ_CP_SERVICE, pfault_interrupt);
248 early_initcall(pfault_irq_init);