1 // SPDX-License-Identifier: GPL-2.0
4 #include <linux/proc_fs.h>
5 #include <linux/seq_file.h>
6 #include <linux/kallsyms.h>
7 #include <linux/utsname.h>
8 #include <linux/security.h>
9 #include <linux/export.h>
10 #include <linux/nospec.h>
12 unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;
13 static struct autogroup autogroup_default;
14 static atomic_t autogroup_seq_nr;
16 void __init autogroup_init(struct task_struct *init_task)
18 autogroup_default.tg = &root_task_group;
19 kref_init(&autogroup_default.kref);
20 init_rwsem(&autogroup_default.lock);
21 init_task->signal->autogroup = &autogroup_default;
24 void autogroup_free(struct task_group *tg)
29 static inline void autogroup_destroy(struct kref *kref)
31 struct autogroup *ag = container_of(kref, struct autogroup, kref);
33 #ifdef CONFIG_RT_GROUP_SCHED
34 /* We've redirected RT tasks to the root task group... */
38 sched_offline_group(ag->tg);
39 sched_destroy_group(ag->tg);
42 static inline void autogroup_kref_put(struct autogroup *ag)
44 kref_put(&ag->kref, autogroup_destroy);
47 static inline struct autogroup *autogroup_kref_get(struct autogroup *ag)
53 static inline struct autogroup *autogroup_task_get(struct task_struct *p)
58 if (!lock_task_sighand(p, &flags))
59 return autogroup_kref_get(&autogroup_default);
61 ag = autogroup_kref_get(p->signal->autogroup);
62 unlock_task_sighand(p, &flags);
67 static inline struct autogroup *autogroup_create(void)
69 struct autogroup *ag = kzalloc(sizeof(*ag), GFP_KERNEL);
70 struct task_group *tg;
75 tg = sched_create_group(&root_task_group);
80 init_rwsem(&ag->lock);
81 ag->id = atomic_inc_return(&autogroup_seq_nr);
83 #ifdef CONFIG_RT_GROUP_SCHED
85 * Autogroup RT tasks are redirected to the root task group
86 * so we don't have to move tasks around upon policy change,
87 * or flail around trying to allocate bandwidth on the fly.
88 * A bandwidth exception in __sched_setscheduler() allows
89 * the policy change to proceed.
91 free_rt_sched_group(tg);
92 tg->rt_se = root_task_group.rt_se;
93 tg->rt_rq = root_task_group.rt_rq;
97 sched_online_group(tg, &root_task_group);
103 if (printk_ratelimit()) {
104 printk(KERN_WARNING "autogroup_create: %s failure.\n",
105 ag ? "sched_create_group()" : "kzalloc()");
108 return autogroup_kref_get(&autogroup_default);
111 bool task_wants_autogroup(struct task_struct *p, struct task_group *tg)
113 if (tg != &root_task_group)
116 * If we race with autogroup_move_group() the caller can use the old
117 * value of signal->autogroup but in this case sched_move_task() will
118 * be called again before autogroup_kref_put().
120 * However, there is no way sched_autogroup_exit_task() could tell us
121 * to avoid autogroup->tg, so we abuse PF_EXITING flag for this case.
123 if (p->flags & PF_EXITING)
129 void sched_autogroup_exit_task(struct task_struct *p)
132 * We are going to call exit_notify() and autogroup_move_group() can't
133 * see this thread after that: we can no longer use signal->autogroup.
134 * See the PF_EXITING check in task_wants_autogroup().
140 autogroup_move_group(struct task_struct *p, struct autogroup *ag)
142 struct autogroup *prev;
143 struct task_struct *t;
146 BUG_ON(!lock_task_sighand(p, &flags));
148 prev = p->signal->autogroup;
150 unlock_task_sighand(p, &flags);
154 p->signal->autogroup = autogroup_kref_get(ag);
156 * We can't avoid sched_move_task() after we changed signal->autogroup,
157 * this process can already run with task_group() == prev->tg or we can
158 * race with cgroup code which can read autogroup = prev under rq->lock.
159 * In the latter case for_each_thread() can not miss a migrating thread,
160 * cpu_cgroup_attach() must not be possible after cgroup_exit() and it
161 * can't be removed from thread list, we hold ->siglock.
163 * If an exiting thread was already removed from thread list we rely on
164 * sched_autogroup_exit_task().
166 for_each_thread(p, t)
169 unlock_task_sighand(p, &flags);
170 autogroup_kref_put(prev);
173 /* Allocates GFP_KERNEL, cannot be called under any spinlock */
174 void sched_autogroup_create_attach(struct task_struct *p)
176 struct autogroup *ag = autogroup_create();
178 autogroup_move_group(p, ag);
179 /* drop extra reference added by autogroup_create() */
180 autogroup_kref_put(ag);
182 EXPORT_SYMBOL(sched_autogroup_create_attach);
184 /* Cannot be called under siglock. Currently has no users */
185 void sched_autogroup_detach(struct task_struct *p)
187 autogroup_move_group(p, &autogroup_default);
189 EXPORT_SYMBOL(sched_autogroup_detach);
191 void sched_autogroup_fork(struct signal_struct *sig)
193 sig->autogroup = autogroup_task_get(current);
196 void sched_autogroup_exit(struct signal_struct *sig)
198 autogroup_kref_put(sig->autogroup);
201 static int __init setup_autogroup(char *str)
203 sysctl_sched_autogroup_enabled = 0;
208 __setup("noautogroup", setup_autogroup);
210 #ifdef CONFIG_PROC_FS
212 int proc_sched_autogroup_set_nice(struct task_struct *p, int nice)
214 static unsigned long next = INITIAL_JIFFIES;
215 struct autogroup *ag;
216 unsigned long shares;
219 if (nice < MIN_NICE || nice > MAX_NICE)
222 err = security_task_setnice(current, nice);
226 if (nice < 0 && !can_nice(current, nice))
229 /* this is a heavy operation taking global locks.. */
230 if (!capable(CAP_SYS_ADMIN) && time_before(jiffies, next))
233 next = HZ / 10 + jiffies;
234 ag = autogroup_task_get(p);
236 idx = array_index_nospec(nice + 20, 40);
237 shares = scale_load(sched_prio_to_weight[idx]);
239 down_write(&ag->lock);
240 err = sched_group_set_shares(ag->tg, shares);
245 autogroup_kref_put(ag);
250 void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m)
252 struct autogroup *ag = autogroup_task_get(p);
254 if (!task_group_is_autogroup(ag->tg))
257 down_read(&ag->lock);
258 seq_printf(m, "/autogroup-%ld nice %d\n", ag->id, ag->nice);
262 autogroup_kref_put(ag);
264 #endif /* CONFIG_PROC_FS */
266 #ifdef CONFIG_SCHED_DEBUG
267 int autogroup_path(struct task_group *tg, char *buf, int buflen)
269 if (!task_group_is_autogroup(tg))
272 return snprintf(buf, buflen, "%s-%ld", "/autogroup", tg->autogroup->id);
274 #endif /* CONFIG_SCHED_DEBUG */