2 #include "sched-pelt.h"
4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
10 #ifdef CONFIG_SCHED_THERMAL_PRESSURE
11 int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
13 static inline u64 thermal_load_avg(struct rq *rq)
15 return READ_ONCE(rq->avg_thermal.load_avg);
19 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
24 static inline u64 thermal_load_avg(struct rq *rq)
30 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
31 int update_irq_load_avg(struct rq *rq, u64 running);
34 update_irq_load_avg(struct rq *rq, u64 running)
40 #define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024)
42 static inline u32 get_pelt_divider(struct sched_avg *avg)
44 return PELT_MIN_DIVIDER + avg->period_contrib;
47 static inline void cfs_se_util_change(struct sched_avg *avg)
49 unsigned int enqueued;
51 if (!sched_feat(UTIL_EST))
54 /* Avoid store if the flag has been already reset */
55 enqueued = avg->util_est.enqueued;
56 if (!(enqueued & UTIL_AVG_UNCHANGED))
59 /* Reset flag to report util_avg has been updated */
60 enqueued &= ~UTIL_AVG_UNCHANGED;
61 WRITE_ONCE(avg->util_est.enqueued, enqueued);
65 * The clock_pelt scales the time to reflect the effective amount of
66 * computation done during the running delta time but then sync back to
67 * clock_task when rq is idle.
70 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
71 * @ max capacity ------******---------------******---------------
72 * @ half capacity ------************---------************---------
73 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16
76 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
78 if (unlikely(is_idle_task(rq->curr))) {
79 /* The rq is idle, we can sync to clock_task */
80 rq->clock_pelt = rq_clock_task(rq);
85 * When a rq runs at a lower compute capacity, it will need
86 * more time to do the same amount of work than at max
87 * capacity. In order to be invariant, we scale the delta to
88 * reflect how much work has been really done.
89 * Running longer results in stealing idle time that will
90 * disturb the load signal compared to max capacity. This
91 * stolen idle time will be automatically reflected when the
92 * rq will be idle and the clock will be synced with
97 * Scale the elapsed time to reflect the real amount of
100 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
101 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
103 rq->clock_pelt += delta;
107 * When rq becomes idle, we have to check if it has lost idle time
108 * because it was fully busy. A rq is fully used when the /Sum util_sum
109 * is greater or equal to:
110 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
111 * For optimization and computing rounding purpose, we don't take into account
112 * the position in the current window (period_contrib) and we use the higher
113 * bound of util_sum to decide.
115 static inline void update_idle_rq_clock_pelt(struct rq *rq)
117 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
118 u32 util_sum = rq->cfs.avg.util_sum;
119 util_sum += rq->avg_rt.util_sum;
120 util_sum += rq->avg_dl.util_sum;
123 * Reflecting stolen time makes sense only if the idle
124 * phase would be present at max capacity. As soon as the
125 * utilization of a rq has reached the maximum value, it is
126 * considered as an always running rq without idle time to
127 * steal. This potential idle time is considered as lost in
128 * this case. We keep track of this lost idle time compare to
131 if (util_sum >= divider)
132 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
135 static inline u64 rq_clock_pelt(struct rq *rq)
137 lockdep_assert_rq_held(rq);
138 assert_clock_updated(rq);
140 return rq->clock_pelt - rq->lost_idle_time;
143 #ifdef CONFIG_CFS_BANDWIDTH
144 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
145 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
147 if (unlikely(cfs_rq->throttle_count))
148 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
150 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
153 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
155 return rq_clock_pelt(rq_of(cfs_rq));
162 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
168 update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
174 update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
180 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
185 static inline u64 thermal_load_avg(struct rq *rq)
191 update_irq_load_avg(struct rq *rq, u64 running)
196 static inline u64 rq_clock_pelt(struct rq *rq)
198 return rq_clock_task(rq);
202 update_rq_clock_pelt(struct rq *rq, s64 delta) { }
205 update_idle_rq_clock_pelt(struct rq *rq) { }