drivers/base/power/domain_governor.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * drivers/base/power/domain_governor.c - Governors for device PM domains.
   4  *
   5  * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/pm_domain.h>
   9 #include <linux/pm_qos.h>
  10 #include <linux/hrtimer.h>
  11 #include <linux/cpuidle.h>
  12 #include <linux/cpumask.h>
  13 #include <linux/ktime.h>
  14
  15 static int dev_update_qos_constraint(struct device *dev, void *data)
  16 {
  17         s64 *constraint_ns_p = data;
  18         s64 constraint_ns;
  19
  20         if (dev->power.subsys_data && dev->power.subsys_data->domain_data) {
  21                 struct gpd_timing_data *td = dev_gpd_data(dev)->td;
  22
  23                 /*
  24                  * Only take suspend-time QoS constraints of devices into
  25                  * account, because constraints updated after the device has
  26                  * been suspended are not guaranteed to be taken into account
  27                  * anyway.  In order for them to take effect, the device has to
  28                  * be resumed and suspended again.
  29                  */
  30                 constraint_ns = td ? td->effective_constraint_ns :
  31                                 PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
  32         } else {
  33                 /*
  34                  * The child is not in a domain and there's no info on its
  35                  * suspend/resume latencies, so assume them to be negligible and
  36                  * take its current PM QoS constraint (that's the only thing
  37                  * known at this point anyway).
  38                  */
  39                 constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY);
  40                 constraint_ns *= NSEC_PER_USEC;
  41         }
  42
  43         if (constraint_ns < *constraint_ns_p)
  44                 *constraint_ns_p = constraint_ns;
  45
  46         return 0;
  47 }
  48
  49 /**
  50  * default_suspend_ok - Default PM domain governor routine to suspend devices.
  51  * @dev: Device to check.
  52  */
  53 static bool default_suspend_ok(struct device *dev)
  54 {
  55         struct gpd_timing_data *td = dev_gpd_data(dev)->td;
  56         unsigned long flags;
  57         s64 constraint_ns;
  58
  59         dev_dbg(dev, "%s()\n", __func__);
  60
  61         spin_lock_irqsave(&dev->power.lock, flags);
  62
  63         if (!td->constraint_changed) {
  64                 bool ret = td->cached_suspend_ok;
  65
  66                 spin_unlock_irqrestore(&dev->power.lock, flags);
  67                 return ret;
  68         }
  69         td->constraint_changed = false;
  70         td->cached_suspend_ok = false;
  71         td->effective_constraint_ns = 0;
  72         constraint_ns = __dev_pm_qos_resume_latency(dev);
  73
  74         spin_unlock_irqrestore(&dev->power.lock, flags);
  75
  76         if (constraint_ns == 0)
  77                 return false;
  78
  79         constraint_ns *= NSEC_PER_USEC;
  80         /*
  81          * We can walk the children without any additional locking, because
  82          * they all have been suspended at this point and their
  83          * effective_constraint_ns fields won't be modified in parallel with us.
  84          */
  85         if (!dev->power.ignore_children)
  86                 device_for_each_child(dev, &constraint_ns,
  87                                       dev_update_qos_constraint);
  88
  89         if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) {
  90                 /* "No restriction", so the device is allowed to suspend. */
  91                 td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
  92                 td->cached_suspend_ok = true;
  93         } else if (constraint_ns == 0) {
  94                 /*
  95                  * This triggers if one of the children that don't belong to a
  96                  * domain has a zero PM QoS constraint and it's better not to
  97                  * suspend then.  effective_constraint_ns is zero already and
  98                  * cached_suspend_ok is false, so bail out.
  99                  */
 100                 return false;
 101         } else {
 102                 constraint_ns -= td->suspend_latency_ns +
 103                                 td->resume_latency_ns;
 104                 /*
 105                  * effective_constraint_ns is zero already and cached_suspend_ok
 106                  * is false, so if the computed value is not positive, return
 107                  * right away.
 108                  */
 109                 if (constraint_ns <= 0)
 110                         return false;
 111
 112                 td->effective_constraint_ns = constraint_ns;
 113                 td->cached_suspend_ok = true;
 114         }
 115
 116         /*
 117          * The children have been suspended already, so we don't need to take
 118          * their suspend latencies into account here.
 119          */
 120         return td->cached_suspend_ok;
 121 }
 122
 123 static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now)
 124 {
 125         ktime_t domain_wakeup = KTIME_MAX;
 126         ktime_t next_wakeup;
 127         struct pm_domain_data *pdd;
 128         struct gpd_link *link;
 129
 130         if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY))
 131                 return;
 132
 133         /*
 134          * Devices that have a predictable wakeup pattern, may specify
 135          * their next wakeup. Let's find the next wakeup from all the
 136          * devices attached to this domain and from all the sub-domains.
 137          * It is possible that component's a next wakeup may have become
 138          * stale when we read that here. We will ignore to ensure the domain
 139          * is able to enter its optimal idle state.
 140          */
 141         list_for_each_entry(pdd, &genpd->dev_list, list_node) {
 142                 next_wakeup = to_gpd_data(pdd)->td->next_wakeup;
 143                 if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))
 144                         if (ktime_before(next_wakeup, domain_wakeup))
 145                                 domain_wakeup = next_wakeup;
 146         }
 147
 148         list_for_each_entry(link, &genpd->parent_links, parent_node) {
 149                 struct genpd_governor_data *cgd = link->child->gd;
 150
 151                 next_wakeup = cgd ? cgd->next_wakeup : KTIME_MAX;
 152                 if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))
 153                         if (ktime_before(next_wakeup, domain_wakeup))
 154                                 domain_wakeup = next_wakeup;
 155         }
 156
 157         genpd->gd->next_wakeup = domain_wakeup;
 158 }
 159
 160 static bool next_wakeup_allows_state(struct generic_pm_domain *genpd,
 161                                      unsigned int state, ktime_t now)
 162 {
 163         ktime_t domain_wakeup = genpd->gd->next_wakeup;
 164         s64 idle_time_ns, min_sleep_ns;
 165
 166         min_sleep_ns = genpd->states[state].power_off_latency_ns +
 167                        genpd->states[state].residency_ns;
 168
 169         idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));
 170
 171         return idle_time_ns >= min_sleep_ns;
 172 }
 173
 174 static bool __default_power_down_ok(struct dev_pm_domain *pd,
 175                                      unsigned int state)
 176 {
 177         struct generic_pm_domain *genpd = pd_to_genpd(pd);
 178         struct gpd_link *link;
 179         struct pm_domain_data *pdd;
 180         s64 min_off_time_ns;
 181         s64 off_on_time_ns;
 182
 183         off_on_time_ns = genpd->states[state].power_off_latency_ns +
 184                 genpd->states[state].power_on_latency_ns;
 185
 186         min_off_time_ns = -1;
 187         /*
 188          * Check if subdomains can be off for enough time.
 189          *
 190          * All subdomains have been powered off already at this point.
 191          */
 192         list_for_each_entry(link, &genpd->parent_links, parent_node) {
 193                 struct genpd_governor_data *cgd = link->child->gd;
 194
 195                 s64 sd_max_off_ns = cgd ? cgd->max_off_time_ns : -1;
 196
 197                 if (sd_max_off_ns < 0)
 198                         continue;
 199
 200                 /*
 201                  * Check if the subdomain is allowed to be off long enough for
 202                  * the current domain to turn off and on (that's how much time
 203                  * it will have to wait worst case).
 204                  */
 205                 if (sd_max_off_ns <= off_on_time_ns)
 206                         return false;
 207
 208                 if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0)
 209                         min_off_time_ns = sd_max_off_ns;
 210         }
 211
 212         /*
 213          * Check if the devices in the domain can be off enough time.
 214          */
 215         list_for_each_entry(pdd, &genpd->dev_list, list_node) {
 216                 struct gpd_timing_data *td;
 217                 s64 constraint_ns;
 218
 219                 /*
 220                  * Check if the device is allowed to be off long enough for the
 221                  * domain to turn off and on (that's how much time it will
 222                  * have to wait worst case).
 223                  */
 224                 td = to_gpd_data(pdd)->td;
 225                 constraint_ns = td->effective_constraint_ns;
 226                 /*
 227                  * Zero means "no suspend at all" and this runs only when all
 228                  * devices in the domain are suspended, so it must be positive.
 229                  */
 230                 if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS)
 231                         continue;
 232
 233                 if (constraint_ns <= off_on_time_ns)
 234                         return false;
 235
 236                 if (min_off_time_ns > constraint_ns || min_off_time_ns < 0)
 237                         min_off_time_ns = constraint_ns;
 238         }
 239
 240         /*
 241          * If the computed minimum device off time is negative, there are no
 242          * latency constraints, so the domain can spend arbitrary time in the
 243          * "off" state.
 244          */
 245         if (min_off_time_ns < 0)
 246                 return true;
 247
 248         /*
 249          * The difference between the computed minimum subdomain or device off
 250          * time and the time needed to turn the domain on is the maximum
 251          * theoretical time this domain can spend in the "off" state.
 252          */
 253         genpd->gd->max_off_time_ns = min_off_time_ns -
 254                 genpd->states[state].power_on_latency_ns;
 255         return true;
 256 }
 257
 258 /**
 259  * _default_power_down_ok - Default generic PM domain power off governor routine.
 260  * @pd: PM domain to check.
 261  * @now: current ktime.
 262  *
 263  * This routine must be executed under the PM domain's lock.
 264  */
 265 static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now)
 266 {
 267         struct generic_pm_domain *genpd = pd_to_genpd(pd);
 268         struct genpd_governor_data *gd = genpd->gd;
 269         int state_idx = genpd->state_count - 1;
 270         struct gpd_link *link;
 271
 272         /*
 273          * Find the next wakeup from devices that can determine their own wakeup
 274          * to find when the domain would wakeup and do it for every device down
 275          * the hierarchy. It is not worth while to sleep if the state's residency
 276          * cannot be met.
 277          */
 278         update_domain_next_wakeup(genpd, now);
 279         if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (gd->next_wakeup != KTIME_MAX)) {
 280                 /* Let's find out the deepest domain idle state, the devices prefer */
 281                 while (state_idx >= 0) {
 282                         if (next_wakeup_allows_state(genpd, state_idx, now)) {
 283                                 gd->max_off_time_changed = true;
 284                                 break;
 285                         }
 286                         state_idx--;
 287                 }
 288
 289                 if (state_idx < 0) {
 290                         state_idx = 0;
 291                         gd->cached_power_down_ok = false;
 292                         goto done;
 293                 }
 294         }
 295
 296         if (!gd->max_off_time_changed) {
 297                 genpd->state_idx = gd->cached_power_down_state_idx;
 298                 return gd->cached_power_down_ok;
 299         }
 300
 301         /*
 302          * We have to invalidate the cached results for the parents, so
 303          * use the observation that default_power_down_ok() is not
 304          * going to be called for any parent until this instance
 305          * returns.
 306          */
 307         list_for_each_entry(link, &genpd->child_links, child_node) {
 308                 struct genpd_governor_data *pgd = link->parent->gd;
 309
 310                 if (pgd)
 311                         pgd->max_off_time_changed = true;
 312         }
 313
 314         gd->max_off_time_ns = -1;
 315         gd->max_off_time_changed = false;
 316         gd->cached_power_down_ok = true;
 317
 318         /*
 319          * Find a state to power down to, starting from the state
 320          * determined by the next wakeup.
 321          */
 322         while (!__default_power_down_ok(pd, state_idx)) {
 323                 if (state_idx == 0) {
 324                         gd->cached_power_down_ok = false;
 325                         break;
 326                 }
 327                 state_idx--;
 328         }
 329
 330 done:
 331         genpd->state_idx = state_idx;
 332         gd->cached_power_down_state_idx = genpd->state_idx;
 333         return gd->cached_power_down_ok;
 334 }
 335
 336 static bool default_power_down_ok(struct dev_pm_domain *pd)
 337 {
 338         return _default_power_down_ok(pd, ktime_get());
 339 }
 340
 341 #ifdef CONFIG_CPU_IDLE
 342 static bool cpu_power_down_ok(struct dev_pm_domain *pd)
 343 {
 344         struct generic_pm_domain *genpd = pd_to_genpd(pd);
 345         struct cpuidle_device *dev;
 346         ktime_t domain_wakeup, next_hrtimer;
 347         ktime_t now = ktime_get();
 348         s64 idle_duration_ns;
 349         int cpu, i;
 350
 351         /* Validate dev PM QoS constraints. */
 352         if (!_default_power_down_ok(pd, now))
 353                 return false;
 354
 355         if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN))
 356                 return true;
 357
 358         /*
 359          * Find the next wakeup for any of the online CPUs within the PM domain
 360          * and its subdomains. Note, we only need the genpd->cpus, as it already
 361          * contains a mask of all CPUs from subdomains.
 362          */
 363         domain_wakeup = ktime_set(KTIME_SEC_MAX, 0);
 364         for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) {
 365                 dev = per_cpu(cpuidle_devices, cpu);
 366                 if (dev) {
 367                         next_hrtimer = READ_ONCE(dev->next_hrtimer);
 368                         if (ktime_before(next_hrtimer, domain_wakeup))
 369                                 domain_wakeup = next_hrtimer;
 370                 }
 371         }
 372
 373         /* The minimum idle duration is from now - until the next wakeup. */
 374         idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));
 375         if (idle_duration_ns <= 0)
 376                 return false;
 377
 378         /*
 379          * Find the deepest idle state that has its residency value satisfied
 380          * and by also taking into account the power off latency for the state.
 381          * Start at the state picked by the dev PM QoS constraint validation.
 382          */
 383         i = genpd->state_idx;
 384         do {
 385                 if (idle_duration_ns >= (genpd->states[i].residency_ns +
 386                     genpd->states[i].power_off_latency_ns)) {
 387                         genpd->state_idx = i;
 388                         return true;
 389                 }
 390         } while (--i >= 0);
 391
 392         return false;
 393 }
 394
 395 struct dev_power_governor pm_domain_cpu_gov = {
 396         .suspend_ok = default_suspend_ok,
 397         .power_down_ok = cpu_power_down_ok,
 398 };
 399 #endif
 400
 401 struct dev_power_governor simple_qos_governor = {
 402         .suspend_ok = default_suspend_ok,
 403         .power_down_ok = default_power_down_ok,
 404 };
 405
 406 /**
 407  * pm_genpd_gov_always_on - A governor implementing an always-on policy
 408  */
 409 struct dev_power_governor pm_domain_always_on_gov = {
 410         .suspend_ok = default_suspend_ok,
 411 };