GNU Linux-libre 4.19.211-gnu1
[releases.git] / kernel / sched / cpufreq_schedutil.c
1 /*
2  * CPUFreq governor based on scheduler-provided CPU utilization data.
3  *
4  * Copyright (C) 2016, Intel Corporation
5  * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11
12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13
14 #include "sched.h"
15
16 #include <trace/events/power.h>
17
18 struct sugov_tunables {
19         struct gov_attr_set     attr_set;
20         unsigned int            rate_limit_us;
21 };
22
23 struct sugov_policy {
24         struct cpufreq_policy   *policy;
25
26         struct sugov_tunables   *tunables;
27         struct list_head        tunables_hook;
28
29         raw_spinlock_t          update_lock;    /* For shared policies */
30         u64                     last_freq_update_time;
31         s64                     freq_update_delay_ns;
32         unsigned int            next_freq;
33         unsigned int            cached_raw_freq;
34
35         /* The next fields are only needed if fast switch cannot be used: */
36         struct                  irq_work irq_work;
37         struct                  kthread_work work;
38         struct                  mutex work_lock;
39         struct                  kthread_worker worker;
40         struct task_struct      *thread;
41         bool                    work_in_progress;
42
43         bool                    limits_changed;
44         bool                    need_freq_update;
45 };
46
47 struct sugov_cpu {
48         struct update_util_data update_util;
49         struct sugov_policy     *sg_policy;
50         unsigned int            cpu;
51
52         bool                    iowait_boost_pending;
53         unsigned int            iowait_boost;
54         u64                     last_update;
55
56         unsigned long           bw_dl;
57         unsigned long           min;
58         unsigned long           max;
59
60         /* The field below is for single-CPU policies only: */
61 #ifdef CONFIG_NO_HZ_COMMON
62         unsigned long           saved_idle_calls;
63 #endif
64 };
65
66 static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);
67
68 /************************ Governor internals ***********************/
69
70 static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
71 {
72         s64 delta_ns;
73
74         /*
75          * Since cpufreq_update_util() is called with rq->lock held for
76          * the @target_cpu, our per-CPU data is fully serialized.
77          *
78          * However, drivers cannot in general deal with cross-CPU
79          * requests, so while get_next_freq() will work, our
80          * sugov_update_commit() call may not for the fast switching platforms.
81          *
82          * Hence stop here for remote requests if they aren't supported
83          * by the hardware, as calculating the frequency is pointless if
84          * we cannot in fact act on it.
85          *
86          * This is needed on the slow switching platforms too to prevent CPUs
87          * going offline from leaving stale IRQ work items behind.
88          */
89         if (!cpufreq_this_cpu_can_update(sg_policy->policy))
90                 return false;
91
92         if (unlikely(sg_policy->limits_changed)) {
93                 sg_policy->limits_changed = false;
94                 sg_policy->need_freq_update = true;
95                 return true;
96         }
97
98         delta_ns = time - sg_policy->last_freq_update_time;
99
100         return delta_ns >= sg_policy->freq_update_delay_ns;
101 }
102
103 static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time,
104                                    unsigned int next_freq)
105 {
106         if (sg_policy->next_freq == next_freq)
107                 return false;
108
109         sg_policy->next_freq = next_freq;
110         sg_policy->last_freq_update_time = time;
111
112         return true;
113 }
114
115 static void sugov_fast_switch(struct sugov_policy *sg_policy, u64 time,
116                               unsigned int next_freq)
117 {
118         struct cpufreq_policy *policy = sg_policy->policy;
119         int cpu;
120
121         if (!sugov_update_next_freq(sg_policy, time, next_freq))
122                 return;
123
124         next_freq = cpufreq_driver_fast_switch(policy, next_freq);
125         if (!next_freq)
126                 return;
127
128         policy->cur = next_freq;
129
130         if (trace_cpu_frequency_enabled()) {
131                 for_each_cpu(cpu, policy->cpus)
132                         trace_cpu_frequency(next_freq, cpu);
133         }
134 }
135
136 static void sugov_deferred_update(struct sugov_policy *sg_policy, u64 time,
137                                   unsigned int next_freq)
138 {
139         if (!sugov_update_next_freq(sg_policy, time, next_freq))
140                 return;
141
142         if (!sg_policy->work_in_progress) {
143                 sg_policy->work_in_progress = true;
144                 irq_work_queue(&sg_policy->irq_work);
145         }
146 }
147
148 /**
149  * get_next_freq - Compute a new frequency for a given cpufreq policy.
150  * @sg_policy: schedutil policy object to compute the new frequency for.
151  * @util: Current CPU utilization.
152  * @max: CPU capacity.
153  *
154  * If the utilization is frequency-invariant, choose the new frequency to be
155  * proportional to it, that is
156  *
157  * next_freq = C * max_freq * util / max
158  *
159  * Otherwise, approximate the would-be frequency-invariant utilization by
160  * util_raw * (curr_freq / max_freq) which leads to
161  *
162  * next_freq = C * curr_freq * util_raw / max
163  *
164  * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
165  *
166  * The lowest driver-supported frequency which is equal or greater than the raw
167  * next_freq (as calculated above) is returned, subject to policy min/max and
168  * cpufreq driver limitations.
169  */
170 static unsigned int get_next_freq(struct sugov_policy *sg_policy,
171                                   unsigned long util, unsigned long max)
172 {
173         struct cpufreq_policy *policy = sg_policy->policy;
174         unsigned int freq = arch_scale_freq_invariant() ?
175                                 policy->cpuinfo.max_freq : policy->cur;
176
177         freq = (freq + (freq >> 2)) * util / max;
178
179         if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)
180                 return sg_policy->next_freq;
181
182         sg_policy->need_freq_update = false;
183         sg_policy->cached_raw_freq = freq;
184         return cpufreq_driver_resolve_freq(policy, freq);
185 }
186
187 /*
188  * This function computes an effective utilization for the given CPU, to be
189  * used for frequency selection given the linear relation: f = u * f_max.
190  *
191  * The scheduler tracks the following metrics:
192  *
193  *   cpu_util_{cfs,rt,dl,irq}()
194  *   cpu_bw_dl()
195  *
196  * Where the cfs,rt and dl util numbers are tracked with the same metric and
197  * synchronized windows and are thus directly comparable.
198  *
199  * The cfs,rt,dl utilization are the running times measured with rq->clock_task
200  * which excludes things like IRQ and steal-time. These latter are then accrued
201  * in the irq utilization.
202  *
203  * The DL bandwidth number otoh is not a measured metric but a value computed
204  * based on the task model parameters and gives the minimal utilization
205  * required to meet deadlines.
206  */
207 static unsigned long sugov_get_util(struct sugov_cpu *sg_cpu)
208 {
209         struct rq *rq = cpu_rq(sg_cpu->cpu);
210         unsigned long util, irq, max;
211
212         sg_cpu->max = max = arch_scale_cpu_capacity(NULL, sg_cpu->cpu);
213         sg_cpu->bw_dl = cpu_bw_dl(rq);
214
215         if (rt_rq_is_runnable(&rq->rt))
216                 return max;
217
218         /*
219          * Early check to see if IRQ/steal time saturates the CPU, can be
220          * because of inaccuracies in how we track these -- see
221          * update_irq_load_avg().
222          */
223         irq = cpu_util_irq(rq);
224         if (unlikely(irq >= max))
225                 return max;
226
227         /*
228          * Because the time spend on RT/DL tasks is visible as 'lost' time to
229          * CFS tasks and we use the same metric to track the effective
230          * utilization (PELT windows are synchronized) we can directly add them
231          * to obtain the CPU's actual utilization.
232          */
233         util = cpu_util_cfs(rq);
234         util += cpu_util_rt(rq);
235
236         /*
237          * We do not make cpu_util_dl() a permanent part of this sum because we
238          * want to use cpu_bw_dl() later on, but we need to check if the
239          * CFS+RT+DL sum is saturated (ie. no idle time) such that we select
240          * f_max when there is no idle time.
241          *
242          * NOTE: numerical errors or stop class might cause us to not quite hit
243          * saturation when we should -- something for later.
244          */
245         if ((util + cpu_util_dl(rq)) >= max)
246                 return max;
247
248         /*
249          * There is still idle time; further improve the number by using the
250          * irq metric. Because IRQ/steal time is hidden from the task clock we
251          * need to scale the task numbers:
252          *
253          *              1 - irq
254          *   U' = irq + ------- * U
255          *                max
256          */
257         util = scale_irq_capacity(util, irq, max);
258         util += irq;
259
260         /*
261          * Bandwidth required by DEADLINE must always be granted while, for
262          * FAIR and RT, we use blocked utilization of IDLE CPUs as a mechanism
263          * to gracefully reduce the frequency when no tasks show up for longer
264          * periods of time.
265          *
266          * Ideally we would like to set bw_dl as min/guaranteed freq and util +
267          * bw_dl as requested freq. However, cpufreq is not yet ready for such
268          * an interface. So, we only do the latter for now.
269          */
270         return min(max, util + sg_cpu->bw_dl);
271 }
272
273 /**
274  * sugov_iowait_reset() - Reset the IO boost status of a CPU.
275  * @sg_cpu: the sugov data for the CPU to boost
276  * @time: the update time from the caller
277  * @set_iowait_boost: true if an IO boost has been requested
278  *
279  * The IO wait boost of a task is disabled after a tick since the last update
280  * of a CPU. If a new IO wait boost is requested after more then a tick, then
281  * we enable the boost starting from the minimum frequency, which improves
282  * energy efficiency by ignoring sporadic wakeups from IO.
283  */
284 static bool sugov_iowait_reset(struct sugov_cpu *sg_cpu, u64 time,
285                                bool set_iowait_boost)
286 {
287         s64 delta_ns = time - sg_cpu->last_update;
288
289         /* Reset boost only if a tick has elapsed since last request */
290         if (delta_ns <= TICK_NSEC)
291                 return false;
292
293         sg_cpu->iowait_boost = set_iowait_boost ? sg_cpu->min : 0;
294         sg_cpu->iowait_boost_pending = set_iowait_boost;
295
296         return true;
297 }
298
299 /**
300  * sugov_iowait_boost() - Updates the IO boost status of a CPU.
301  * @sg_cpu: the sugov data for the CPU to boost
302  * @time: the update time from the caller
303  * @flags: SCHED_CPUFREQ_IOWAIT if the task is waking up after an IO wait
304  *
305  * Each time a task wakes up after an IO operation, the CPU utilization can be
306  * boosted to a certain utilization which doubles at each "frequent and
307  * successive" wakeup from IO, ranging from the utilization of the minimum
308  * OPP to the utilization of the maximum OPP.
309  * To keep doubling, an IO boost has to be requested at least once per tick,
310  * otherwise we restart from the utilization of the minimum OPP.
311  */
312 static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
313                                unsigned int flags)
314 {
315         bool set_iowait_boost = flags & SCHED_CPUFREQ_IOWAIT;
316
317         /* Reset boost if the CPU appears to have been idle enough */
318         if (sg_cpu->iowait_boost &&
319             sugov_iowait_reset(sg_cpu, time, set_iowait_boost))
320                 return;
321
322         /* Boost only tasks waking up after IO */
323         if (!set_iowait_boost)
324                 return;
325
326         /* Ensure boost doubles only one time at each request */
327         if (sg_cpu->iowait_boost_pending)
328                 return;
329         sg_cpu->iowait_boost_pending = true;
330
331         /* Double the boost at each request */
332         if (sg_cpu->iowait_boost) {
333                 sg_cpu->iowait_boost =
334                         min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE);
335                 return;
336         }
337
338         /* First wakeup after IO: start with minimum boost */
339         sg_cpu->iowait_boost = sg_cpu->min;
340 }
341
342 /**
343  * sugov_iowait_apply() - Apply the IO boost to a CPU.
344  * @sg_cpu: the sugov data for the cpu to boost
345  * @time: the update time from the caller
346  * @util: the utilization to (eventually) boost
347  * @max: the maximum value the utilization can be boosted to
348  *
349  * A CPU running a task which woken up after an IO operation can have its
350  * utilization boosted to speed up the completion of those IO operations.
351  * The IO boost value is increased each time a task wakes up from IO, in
352  * sugov_iowait_apply(), and it's instead decreased by this function,
353  * each time an increase has not been requested (!iowait_boost_pending).
354  *
355  * A CPU which also appears to have been idle for at least one tick has also
356  * its IO boost utilization reset.
357  *
358  * This mechanism is designed to boost high frequently IO waiting tasks, while
359  * being more conservative on tasks which does sporadic IO operations.
360  */
361 static unsigned long sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time,
362                                         unsigned long util, unsigned long max)
363 {
364         unsigned long boost;
365
366         /* No boost currently required */
367         if (!sg_cpu->iowait_boost)
368                 return util;
369
370         /* Reset boost if the CPU appears to have been idle enough */
371         if (sugov_iowait_reset(sg_cpu, time, false))
372                 return util;
373
374         if (!sg_cpu->iowait_boost_pending) {
375                 /*
376                  * No boost pending; reduce the boost value.
377                  */
378                 sg_cpu->iowait_boost >>= 1;
379                 if (sg_cpu->iowait_boost < sg_cpu->min) {
380                         sg_cpu->iowait_boost = 0;
381                         return util;
382                 }
383         }
384
385         sg_cpu->iowait_boost_pending = false;
386
387         /*
388          * @util is already in capacity scale; convert iowait_boost
389          * into the same scale so we can compare.
390          */
391         boost = (sg_cpu->iowait_boost * max) >> SCHED_CAPACITY_SHIFT;
392         return max(boost, util);
393 }
394
395 #ifdef CONFIG_NO_HZ_COMMON
396 static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)
397 {
398         unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);
399         bool ret = idle_calls == sg_cpu->saved_idle_calls;
400
401         sg_cpu->saved_idle_calls = idle_calls;
402         return ret;
403 }
404 #else
405 static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }
406 #endif /* CONFIG_NO_HZ_COMMON */
407
408 /*
409  * Make sugov_should_update_freq() ignore the rate limit when DL
410  * has increased the utilization.
411  */
412 static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu, struct sugov_policy *sg_policy)
413 {
414         if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_dl)
415                 sg_policy->limits_changed = true;
416 }
417
418 static void sugov_update_single(struct update_util_data *hook, u64 time,
419                                 unsigned int flags)
420 {
421         struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
422         struct sugov_policy *sg_policy = sg_cpu->sg_policy;
423         unsigned long util, max;
424         unsigned int next_f;
425         bool busy;
426
427         sugov_iowait_boost(sg_cpu, time, flags);
428         sg_cpu->last_update = time;
429
430         ignore_dl_rate_limit(sg_cpu, sg_policy);
431
432         if (!sugov_should_update_freq(sg_policy, time))
433                 return;
434
435         /* Limits may have changed, don't skip frequency update */
436         busy = !sg_policy->need_freq_update && sugov_cpu_is_busy(sg_cpu);
437
438         util = sugov_get_util(sg_cpu);
439         max = sg_cpu->max;
440         util = sugov_iowait_apply(sg_cpu, time, util, max);
441         next_f = get_next_freq(sg_policy, util, max);
442         /*
443          * Do not reduce the frequency if the CPU has not been idle
444          * recently, as the reduction is likely to be premature then.
445          */
446         if (busy && next_f < sg_policy->next_freq) {
447                 next_f = sg_policy->next_freq;
448
449                 /* Reset cached freq as next_freq has changed */
450                 sg_policy->cached_raw_freq = 0;
451         }
452
453         /*
454          * This code runs under rq->lock for the target CPU, so it won't run
455          * concurrently on two different CPUs for the same target and it is not
456          * necessary to acquire the lock in the fast switch case.
457          */
458         if (sg_policy->policy->fast_switch_enabled) {
459                 sugov_fast_switch(sg_policy, time, next_f);
460         } else {
461                 raw_spin_lock(&sg_policy->update_lock);
462                 sugov_deferred_update(sg_policy, time, next_f);
463                 raw_spin_unlock(&sg_policy->update_lock);
464         }
465 }
466
467 static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
468 {
469         struct sugov_policy *sg_policy = sg_cpu->sg_policy;
470         struct cpufreq_policy *policy = sg_policy->policy;
471         unsigned long util = 0, max = 1;
472         unsigned int j;
473
474         for_each_cpu(j, policy->cpus) {
475                 struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
476                 unsigned long j_util, j_max;
477
478                 j_util = sugov_get_util(j_sg_cpu);
479                 j_max = j_sg_cpu->max;
480                 j_util = sugov_iowait_apply(j_sg_cpu, time, j_util, j_max);
481
482                 if (j_util * max > j_max * util) {
483                         util = j_util;
484                         max = j_max;
485                 }
486         }
487
488         return get_next_freq(sg_policy, util, max);
489 }
490
491 static void
492 sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags)
493 {
494         struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
495         struct sugov_policy *sg_policy = sg_cpu->sg_policy;
496         unsigned int next_f;
497
498         raw_spin_lock(&sg_policy->update_lock);
499
500         sugov_iowait_boost(sg_cpu, time, flags);
501         sg_cpu->last_update = time;
502
503         ignore_dl_rate_limit(sg_cpu, sg_policy);
504
505         if (sugov_should_update_freq(sg_policy, time)) {
506                 next_f = sugov_next_freq_shared(sg_cpu, time);
507
508                 if (sg_policy->policy->fast_switch_enabled)
509                         sugov_fast_switch(sg_policy, time, next_f);
510                 else
511                         sugov_deferred_update(sg_policy, time, next_f);
512         }
513
514         raw_spin_unlock(&sg_policy->update_lock);
515 }
516
517 static void sugov_work(struct kthread_work *work)
518 {
519         struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);
520         unsigned int freq;
521         unsigned long flags;
522
523         /*
524          * Hold sg_policy->update_lock shortly to handle the case where:
525          * incase sg_policy->next_freq is read here, and then updated by
526          * sugov_deferred_update() just before work_in_progress is set to false
527          * here, we may miss queueing the new update.
528          *
529          * Note: If a work was queued after the update_lock is released,
530          * sugov_work() will just be called again by kthread_work code; and the
531          * request will be proceed before the sugov thread sleeps.
532          */
533         raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
534         freq = sg_policy->next_freq;
535         sg_policy->work_in_progress = false;
536         raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
537
538         mutex_lock(&sg_policy->work_lock);
539         __cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);
540         mutex_unlock(&sg_policy->work_lock);
541 }
542
543 static void sugov_irq_work(struct irq_work *irq_work)
544 {
545         struct sugov_policy *sg_policy;
546
547         sg_policy = container_of(irq_work, struct sugov_policy, irq_work);
548
549         kthread_queue_work(&sg_policy->worker, &sg_policy->work);
550 }
551
552 /************************** sysfs interface ************************/
553
554 static struct sugov_tunables *global_tunables;
555 static DEFINE_MUTEX(global_tunables_lock);
556
557 static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr_set)
558 {
559         return container_of(attr_set, struct sugov_tunables, attr_set);
560 }
561
562 static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf)
563 {
564         struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
565
566         return sprintf(buf, "%u\n", tunables->rate_limit_us);
567 }
568
569 static ssize_t
570 rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count)
571 {
572         struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
573         struct sugov_policy *sg_policy;
574         unsigned int rate_limit_us;
575
576         if (kstrtouint(buf, 10, &rate_limit_us))
577                 return -EINVAL;
578
579         tunables->rate_limit_us = rate_limit_us;
580
581         list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)
582                 sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;
583
584         return count;
585 }
586
587 static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us);
588
589 static struct attribute *sugov_attributes[] = {
590         &rate_limit_us.attr,
591         NULL
592 };
593
594 static void sugov_tunables_free(struct kobject *kobj)
595 {
596         struct gov_attr_set *attr_set = container_of(kobj, struct gov_attr_set, kobj);
597
598         kfree(to_sugov_tunables(attr_set));
599 }
600
601 static struct kobj_type sugov_tunables_ktype = {
602         .default_attrs = sugov_attributes,
603         .sysfs_ops = &governor_sysfs_ops,
604         .release = &sugov_tunables_free,
605 };
606
607 /********************** cpufreq governor interface *********************/
608
609 static struct cpufreq_governor schedutil_gov;
610
611 static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy)
612 {
613         struct sugov_policy *sg_policy;
614
615         sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL);
616         if (!sg_policy)
617                 return NULL;
618
619         sg_policy->policy = policy;
620         raw_spin_lock_init(&sg_policy->update_lock);
621         return sg_policy;
622 }
623
624 static void sugov_policy_free(struct sugov_policy *sg_policy)
625 {
626         kfree(sg_policy);
627 }
628
629 static int sugov_kthread_create(struct sugov_policy *sg_policy)
630 {
631         struct task_struct *thread;
632         struct sched_attr attr = {
633                 .size           = sizeof(struct sched_attr),
634                 .sched_policy   = SCHED_DEADLINE,
635                 .sched_flags    = SCHED_FLAG_SUGOV,
636                 .sched_nice     = 0,
637                 .sched_priority = 0,
638                 /*
639                  * Fake (unused) bandwidth; workaround to "fix"
640                  * priority inheritance.
641                  */
642                 .sched_runtime  =  1000000,
643                 .sched_deadline = 10000000,
644                 .sched_period   = 10000000,
645         };
646         struct cpufreq_policy *policy = sg_policy->policy;
647         int ret;
648
649         /* kthread only required for slow path */
650         if (policy->fast_switch_enabled)
651                 return 0;
652
653         kthread_init_work(&sg_policy->work, sugov_work);
654         kthread_init_worker(&sg_policy->worker);
655         thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
656                                 "sugov:%d",
657                                 cpumask_first(policy->related_cpus));
658         if (IS_ERR(thread)) {
659                 pr_err("failed to create sugov thread: %ld\n", PTR_ERR(thread));
660                 return PTR_ERR(thread);
661         }
662
663         ret = sched_setattr_nocheck(thread, &attr);
664         if (ret) {
665                 kthread_stop(thread);
666                 pr_warn("%s: failed to set SCHED_DEADLINE\n", __func__);
667                 return ret;
668         }
669
670         sg_policy->thread = thread;
671         kthread_bind_mask(thread, policy->related_cpus);
672         init_irq_work(&sg_policy->irq_work, sugov_irq_work);
673         mutex_init(&sg_policy->work_lock);
674
675         wake_up_process(thread);
676
677         return 0;
678 }
679
680 static void sugov_kthread_stop(struct sugov_policy *sg_policy)
681 {
682         /* kthread only required for slow path */
683         if (sg_policy->policy->fast_switch_enabled)
684                 return;
685
686         kthread_flush_worker(&sg_policy->worker);
687         kthread_stop(sg_policy->thread);
688         mutex_destroy(&sg_policy->work_lock);
689 }
690
691 static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)
692 {
693         struct sugov_tunables *tunables;
694
695         tunables = kzalloc(sizeof(*tunables), GFP_KERNEL);
696         if (tunables) {
697                 gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);
698                 if (!have_governor_per_policy())
699                         global_tunables = tunables;
700         }
701         return tunables;
702 }
703
704 static void sugov_clear_global_tunables(void)
705 {
706         if (!have_governor_per_policy())
707                 global_tunables = NULL;
708 }
709
710 static int sugov_init(struct cpufreq_policy *policy)
711 {
712         struct sugov_policy *sg_policy;
713         struct sugov_tunables *tunables;
714         int ret = 0;
715
716         /* State should be equivalent to EXIT */
717         if (policy->governor_data)
718                 return -EBUSY;
719
720         cpufreq_enable_fast_switch(policy);
721
722         sg_policy = sugov_policy_alloc(policy);
723         if (!sg_policy) {
724                 ret = -ENOMEM;
725                 goto disable_fast_switch;
726         }
727
728         ret = sugov_kthread_create(sg_policy);
729         if (ret)
730                 goto free_sg_policy;
731
732         mutex_lock(&global_tunables_lock);
733
734         if (global_tunables) {
735                 if (WARN_ON(have_governor_per_policy())) {
736                         ret = -EINVAL;
737                         goto stop_kthread;
738                 }
739                 policy->governor_data = sg_policy;
740                 sg_policy->tunables = global_tunables;
741
742                 gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);
743                 goto out;
744         }
745
746         tunables = sugov_tunables_alloc(sg_policy);
747         if (!tunables) {
748                 ret = -ENOMEM;
749                 goto stop_kthread;
750         }
751
752         tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);
753
754         policy->governor_data = sg_policy;
755         sg_policy->tunables = tunables;
756
757         ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
758                                    get_governor_parent_kobj(policy), "%s",
759                                    schedutil_gov.name);
760         if (ret)
761                 goto fail;
762
763 out:
764         mutex_unlock(&global_tunables_lock);
765         return 0;
766
767 fail:
768         kobject_put(&tunables->attr_set.kobj);
769         policy->governor_data = NULL;
770         sugov_clear_global_tunables();
771
772 stop_kthread:
773         sugov_kthread_stop(sg_policy);
774         mutex_unlock(&global_tunables_lock);
775
776 free_sg_policy:
777         sugov_policy_free(sg_policy);
778
779 disable_fast_switch:
780         cpufreq_disable_fast_switch(policy);
781
782         pr_err("initialization failed (error %d)\n", ret);
783         return ret;
784 }
785
786 static void sugov_exit(struct cpufreq_policy *policy)
787 {
788         struct sugov_policy *sg_policy = policy->governor_data;
789         struct sugov_tunables *tunables = sg_policy->tunables;
790         unsigned int count;
791
792         mutex_lock(&global_tunables_lock);
793
794         count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
795         policy->governor_data = NULL;
796         if (!count)
797                 sugov_clear_global_tunables();
798
799         mutex_unlock(&global_tunables_lock);
800
801         sugov_kthread_stop(sg_policy);
802         sugov_policy_free(sg_policy);
803         cpufreq_disable_fast_switch(policy);
804 }
805
806 static int sugov_start(struct cpufreq_policy *policy)
807 {
808         struct sugov_policy *sg_policy = policy->governor_data;
809         unsigned int cpu;
810
811         sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
812         sg_policy->last_freq_update_time        = 0;
813         sg_policy->next_freq                    = 0;
814         sg_policy->work_in_progress             = false;
815         sg_policy->limits_changed               = false;
816         sg_policy->need_freq_update             = false;
817         sg_policy->cached_raw_freq              = 0;
818
819         for_each_cpu(cpu, policy->cpus) {
820                 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
821
822                 memset(sg_cpu, 0, sizeof(*sg_cpu));
823                 sg_cpu->cpu                     = cpu;
824                 sg_cpu->sg_policy               = sg_policy;
825                 sg_cpu->min                     =
826                         (SCHED_CAPACITY_SCALE * policy->cpuinfo.min_freq) /
827                         policy->cpuinfo.max_freq;
828         }
829
830         for_each_cpu(cpu, policy->cpus) {
831                 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
832
833                 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
834                                              policy_is_shared(policy) ?
835                                                         sugov_update_shared :
836                                                         sugov_update_single);
837         }
838         return 0;
839 }
840
841 static void sugov_stop(struct cpufreq_policy *policy)
842 {
843         struct sugov_policy *sg_policy = policy->governor_data;
844         unsigned int cpu;
845
846         for_each_cpu(cpu, policy->cpus)
847                 cpufreq_remove_update_util_hook(cpu);
848
849         synchronize_sched();
850
851         if (!policy->fast_switch_enabled) {
852                 irq_work_sync(&sg_policy->irq_work);
853                 kthread_cancel_work_sync(&sg_policy->work);
854         }
855 }
856
857 static void sugov_limits(struct cpufreq_policy *policy)
858 {
859         struct sugov_policy *sg_policy = policy->governor_data;
860
861         if (!policy->fast_switch_enabled) {
862                 mutex_lock(&sg_policy->work_lock);
863                 cpufreq_policy_apply_limits(policy);
864                 mutex_unlock(&sg_policy->work_lock);
865         }
866
867         sg_policy->limits_changed = true;
868 }
869
870 static struct cpufreq_governor schedutil_gov = {
871         .name                   = "schedutil",
872         .owner                  = THIS_MODULE,
873         .dynamic_switching      = true,
874         .init                   = sugov_init,
875         .exit                   = sugov_exit,
876         .start                  = sugov_start,
877         .stop                   = sugov_stop,
878         .limits                 = sugov_limits,
879 };
880
881 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL
882 struct cpufreq_governor *cpufreq_default_governor(void)
883 {
884         return &schedutil_gov;
885 }
886 #endif
887
888 static int __init sugov_register(void)
889 {
890         return cpufreq_register_governor(&schedutil_gov);
891 }
892 fs_initcall(sugov_register);