block/blk-iocost.c

   1 /* SPDX-License-Identifier: GPL-2.0
   2  *
   3  * IO cost model based controller.
   4  *
   5  * Copyright (C) 2019 Tejun Heo <tj@kernel.org>
   6  * Copyright (C) 2019 Andy Newell <newella@fb.com>
   7  * Copyright (C) 2019 Facebook
   8  *
   9  * One challenge of controlling IO resources is the lack of trivially
  10  * observable cost metric.  This is distinguished from CPU and memory where
  11  * wallclock time and the number of bytes can serve as accurate enough
  12  * approximations.
  13  *
  14  * Bandwidth and iops are the most commonly used metrics for IO devices but
  15  * depending on the type and specifics of the device, different IO patterns
  16  * easily lead to multiple orders of magnitude variations rendering them
  17  * useless for the purpose of IO capacity distribution.  While on-device
  18  * time, with a lot of clutches, could serve as a useful approximation for
  19  * non-queued rotational devices, this is no longer viable with modern
  20  * devices, even the rotational ones.
  21  *
  22  * While there is no cost metric we can trivially observe, it isn't a
  23  * complete mystery.  For example, on a rotational device, seek cost
  24  * dominates while a contiguous transfer contributes a smaller amount
  25  * proportional to the size.  If we can characterize at least the relative
  26  * costs of these different types of IOs, it should be possible to
  27  * implement a reasonable work-conserving proportional IO resource
  28  * distribution.
  29  *
  30  * 1. IO Cost Model
  31  *
  32  * IO cost model estimates the cost of an IO given its basic parameters and
  33  * history (e.g. the end sector of the last IO).  The cost is measured in
  34  * device time.  If a given IO is estimated to cost 10ms, the device should
  35  * be able to process ~100 of those IOs in a second.
  36  *
  37  * Currently, there's only one builtin cost model - linear.  Each IO is
  38  * classified as sequential or random and given a base cost accordingly.
  39  * On top of that, a size cost proportional to the length of the IO is
  40  * added.  While simple, this model captures the operational
  41  * characteristics of a wide varienty of devices well enough.  Default
  42  * paramters for several different classes of devices are provided and the
  43  * parameters can be configured from userspace via
  44  * /sys/fs/cgroup/io.cost.model.
  45  *
  46  * If needed, tools/cgroup/iocost_coef_gen.py can be used to generate
  47  * device-specific coefficients.
  48  *
  49  * If needed, tools/cgroup/iocost_coef_gen.py can be used to generate
  50  * device-specific coefficients.
  51  *
  52  * 2. Control Strategy
  53  *
  54  * The device virtual time (vtime) is used as the primary control metric.
  55  * The control strategy is composed of the following three parts.
  56  *
  57  * 2-1. Vtime Distribution
  58  *
  59  * When a cgroup becomes active in terms of IOs, its hierarchical share is
  60  * calculated.  Please consider the following hierarchy where the numbers
  61  * inside parentheses denote the configured weights.
  62  *
  63  *           root
  64  *         /       \
  65  *      A (w:100)  B (w:300)
  66  *      /       \
  67  *  A0 (w:100)  A1 (w:100)
  68  *
  69  * If B is idle and only A0 and A1 are actively issuing IOs, as the two are
  70  * of equal weight, each gets 50% share.  If then B starts issuing IOs, B
  71  * gets 300/(100+300) or 75% share, and A0 and A1 equally splits the rest,
  72  * 12.5% each.  The distribution mechanism only cares about these flattened
  73  * shares.  They're called hweights (hierarchical weights) and always add
  74  * upto 1 (HWEIGHT_WHOLE).
  75  *
  76  * A given cgroup's vtime runs slower in inverse proportion to its hweight.
  77  * For example, with 12.5% weight, A0's time runs 8 times slower (100/12.5)
  78  * against the device vtime - an IO which takes 10ms on the underlying
  79  * device is considered to take 80ms on A0.
  80  *
  81  * This constitutes the basis of IO capacity distribution.  Each cgroup's
  82  * vtime is running at a rate determined by its hweight.  A cgroup tracks
  83  * the vtime consumed by past IOs and can issue a new IO iff doing so
  84  * wouldn't outrun the current device vtime.  Otherwise, the IO is
  85  * suspended until the vtime has progressed enough to cover it.
  86  *
  87  * 2-2. Vrate Adjustment
  88  *
  89  * It's unrealistic to expect the cost model to be perfect.  There are too
  90  * many devices and even on the same device the overall performance
  91  * fluctuates depending on numerous factors such as IO mixture and device
  92  * internal garbage collection.  The controller needs to adapt dynamically.
  93  *
  94  * This is achieved by adjusting the overall IO rate according to how busy
  95  * the device is.  If the device becomes overloaded, we're sending down too
  96  * many IOs and should generally slow down.  If there are waiting issuers
  97  * but the device isn't saturated, we're issuing too few and should
  98  * generally speed up.
  99  *
 100  * To slow down, we lower the vrate - the rate at which the device vtime
 101  * passes compared to the wall clock.  For example, if the vtime is running
 102  * at the vrate of 75%, all cgroups added up would only be able to issue
 103  * 750ms worth of IOs per second, and vice-versa for speeding up.
 104  *
 105  * Device business is determined using two criteria - rq wait and
 106  * completion latencies.
 107  *
 108  * When a device gets saturated, the on-device and then the request queues
 109  * fill up and a bio which is ready to be issued has to wait for a request
 110  * to become available.  When this delay becomes noticeable, it's a clear
 111  * indication that the device is saturated and we lower the vrate.  This
 112  * saturation signal is fairly conservative as it only triggers when both
 113  * hardware and software queues are filled up, and is used as the default
 114  * busy signal.
 115  *
 116  * As devices can have deep queues and be unfair in how the queued commands
 117  * are executed, soley depending on rq wait may not result in satisfactory
 118  * control quality.  For a better control quality, completion latency QoS
 119  * parameters can be configured so that the device is considered saturated
 120  * if N'th percentile completion latency rises above the set point.
 121  *
 122  * The completion latency requirements are a function of both the
 123  * underlying device characteristics and the desired IO latency quality of
 124  * service.  There is an inherent trade-off - the tighter the latency QoS,
 125  * the higher the bandwidth lossage.  Latency QoS is disabled by default
 126  * and can be set through /sys/fs/cgroup/io.cost.qos.
 127  *
 128  * 2-3. Work Conservation
 129  *
 130  * Imagine two cgroups A and B with equal weights.  A is issuing a small IO
 131  * periodically while B is sending out enough parallel IOs to saturate the
 132  * device on its own.  Let's say A's usage amounts to 100ms worth of IO
 133  * cost per second, i.e., 10% of the device capacity.  The naive
 134  * distribution of half and half would lead to 60% utilization of the
 135  * device, a significant reduction in the total amount of work done
 136  * compared to free-for-all competition.  This is too high a cost to pay
 137  * for IO control.
 138  *
 139  * To conserve the total amount of work done, we keep track of how much
 140  * each active cgroup is actually using and yield part of its weight if
 141  * there are other cgroups which can make use of it.  In the above case,
 142  * A's weight will be lowered so that it hovers above the actual usage and
 143  * B would be able to use the rest.
 144  *
 145  * As we don't want to penalize a cgroup for donating its weight, the
 146  * surplus weight adjustment factors in a margin and has an immediate
 147  * snapback mechanism in case the cgroup needs more IO vtime for itself.
 148  *
 149  * Note that adjusting down surplus weights has the same effects as
 150  * accelerating vtime for other cgroups and work conservation can also be
 151  * implemented by adjusting vrate dynamically.  However, squaring who can
 152  * donate and should take back how much requires hweight propagations
 153  * anyway making it easier to implement and understand as a separate
 154  * mechanism.
 155  *
 156  * 3. Monitoring
 157  *
 158  * Instead of debugfs or other clumsy monitoring mechanisms, this
 159  * controller uses a drgn based monitoring script -
 160  * tools/cgroup/iocost_monitor.py.  For details on drgn, please see
 161  * https://github.com/osandov/drgn.  The ouput looks like the following.
 162  *
 163  *  sdb RUN   per=300ms cur_per=234.218:v203.695 busy= +1 vrate= 62.12%
 164  *                 active      weight      hweight% inflt% dbt  delay usages%
 165  *  test/a              *    50/   50  33.33/ 33.33  27.65   2  0*041 033:033:033
 166  *  test/b              *   100/  100  66.67/ 66.67  17.56   0  0*000 066:079:077
 167  *
 168  * - per        : Timer period
 169  * - cur_per    : Internal wall and device vtime clock
 170  * - vrate      : Device virtual time rate against wall clock
 171  * - weight     : Surplus-adjusted and configured weights
 172  * - hweight    : Surplus-adjusted and configured hierarchical weights
 173  * - inflt      : The percentage of in-flight IO cost at the end of last period
 174  * - del_ms     : Deferred issuer delay induction level and duration
 175  * - usages     : Usage history
 176  */
 177
 178 #include <linux/kernel.h>
 179 #include <linux/module.h>
 180 #include <linux/timer.h>
 181 #include <linux/time64.h>
 182 #include <linux/parser.h>
 183 #include <linux/sched/signal.h>
 184 #include <linux/blk-cgroup.h>
 185 #include "blk-rq-qos.h"
 186 #include "blk-stat.h"
 187 #include "blk-wbt.h"
 188
 189 #ifdef CONFIG_TRACEPOINTS
 190
 191 /* copied from TRACE_CGROUP_PATH, see cgroup-internal.h */
 192 #define TRACE_IOCG_PATH_LEN 1024
 193 static DEFINE_SPINLOCK(trace_iocg_path_lock);
 194 static char trace_iocg_path[TRACE_IOCG_PATH_LEN];
 195
 196 #define TRACE_IOCG_PATH(type, iocg, ...)                                        \
 197         do {                                                                    \
 198                 unsigned long flags;                                            \
 199                 if (trace_iocost_##type##_enabled()) {                          \
 200                         spin_lock_irqsave(&trace_iocg_path_lock, flags);        \
 201                         cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup,      \
 202                                     trace_iocg_path, TRACE_IOCG_PATH_LEN);      \
 203                         trace_iocost_##type(iocg, trace_iocg_path,              \
 204                                               ##__VA_ARGS__);                   \
 205                         spin_unlock_irqrestore(&trace_iocg_path_lock, flags);   \
 206                 }                                                               \
 207         } while (0)
 208
 209 #else   /* CONFIG_TRACE_POINTS */
 210 #define TRACE_IOCG_PATH(type, iocg, ...)        do { } while (0)
 211 #endif  /* CONFIG_TRACE_POINTS */
 212
 213 enum {
 214         MILLION                 = 1000000,
 215
 216         /* timer period is calculated from latency requirements, bound it */
 217         MIN_PERIOD              = USEC_PER_MSEC,
 218         MAX_PERIOD              = USEC_PER_SEC,
 219
 220         /*
 221          * A cgroup's vtime can run 50% behind the device vtime, which
 222          * serves as its IO credit buffer.  Surplus weight adjustment is
 223          * immediately canceled if the vtime margin runs below 10%.
 224          */
 225         MARGIN_PCT              = 50,
 226         INUSE_MARGIN_PCT        = 10,
 227
 228         /* Have some play in waitq timer operations */
 229         WAITQ_TIMER_MARGIN_PCT  = 5,
 230
 231         /*
 232          * vtime can wrap well within a reasonable uptime when vrate is
 233          * consistently raised.  Don't trust recorded cgroup vtime if the
 234          * period counter indicates that it's older than 5mins.
 235          */
 236         VTIME_VALID_DUR         = 300 * USEC_PER_SEC,
 237
 238         /*
 239          * Remember the past three non-zero usages and use the max for
 240          * surplus calculation.  Three slots guarantee that we remember one
 241          * full period usage from the last active stretch even after
 242          * partial deactivation and re-activation periods.  Don't start
 243          * giving away weight before collecting two data points to prevent
 244          * hweight adjustments based on one partial activation period.
 245          */
 246         NR_USAGE_SLOTS          = 3,
 247         MIN_VALID_USAGES        = 2,
 248
 249         /* 1/64k is granular enough and can easily be handled w/ u32 */
 250         HWEIGHT_WHOLE           = 1 << 16,
 251
 252         /*
 253          * As vtime is used to calculate the cost of each IO, it needs to
 254          * be fairly high precision.  For example, it should be able to
 255          * represent the cost of a single page worth of discard with
 256          * suffificient accuracy.  At the same time, it should be able to
 257          * represent reasonably long enough durations to be useful and
 258          * convenient during operation.
 259          *
 260          * 1s worth of vtime is 2^37.  This gives us both sub-nanosecond
 261          * granularity and days of wrap-around time even at extreme vrates.
 262          */
 263         VTIME_PER_SEC_SHIFT     = 37,
 264         VTIME_PER_SEC           = 1LLU << VTIME_PER_SEC_SHIFT,
 265         VTIME_PER_USEC          = VTIME_PER_SEC / USEC_PER_SEC,
 266
 267         /* bound vrate adjustments within two orders of magnitude */
 268         VRATE_MIN_PPM           = 10000,        /* 1% */
 269         VRATE_MAX_PPM           = 100000000,    /* 10000% */
 270
 271         VRATE_MIN               = VTIME_PER_USEC * VRATE_MIN_PPM / MILLION,
 272         VRATE_CLAMP_ADJ_PCT     = 4,
 273
 274         /* if IOs end up waiting for requests, issue less */
 275         RQ_WAIT_BUSY_PCT        = 5,
 276
 277         /* unbusy hysterisis */
 278         UNBUSY_THR_PCT          = 75,
 279
 280         /* don't let cmds which take a very long time pin lagging for too long */
 281         MAX_LAGGING_PERIODS     = 10,
 282
 283         /*
 284          * If usage% * 1.25 + 2% is lower than hweight% by more than 3%,
 285          * donate the surplus.
 286          */
 287         SURPLUS_SCALE_PCT       = 125,                  /* * 125% */
 288         SURPLUS_SCALE_ABS       = HWEIGHT_WHOLE / 50,   /* + 2% */
 289         SURPLUS_MIN_ADJ_DELTA   = HWEIGHT_WHOLE / 33,   /* 3% */
 290
 291         /* switch iff the conditions are met for longer than this */
 292         AUTOP_CYCLE_NSEC        = 10LLU * NSEC_PER_SEC,
 293
 294         /*
 295          * Count IO size in 4k pages.  The 12bit shift helps keeping
 296          * size-proportional components of cost calculation in closer
 297          * numbers of digits to per-IO cost components.
 298          */
 299         IOC_PAGE_SHIFT          = 12,
 300         IOC_PAGE_SIZE           = 1 << IOC_PAGE_SHIFT,
 301         IOC_SECT_TO_PAGE_SHIFT  = IOC_PAGE_SHIFT - SECTOR_SHIFT,
 302
 303         /* if apart further than 16M, consider randio for linear model */
 304         LCOEF_RANDIO_PAGES      = 4096,
 305 };
 306
 307 enum ioc_running {
 308         IOC_IDLE,
 309         IOC_RUNNING,
 310         IOC_STOP,
 311 };
 312
 313 /* io.cost.qos controls including per-dev enable of the whole controller */
 314 enum {
 315         QOS_ENABLE,
 316         QOS_CTRL,
 317         NR_QOS_CTRL_PARAMS,
 318 };
 319
 320 /* io.cost.qos params */
 321 enum {
 322         QOS_RPPM,
 323         QOS_RLAT,
 324         QOS_WPPM,
 325         QOS_WLAT,
 326         QOS_MIN,
 327         QOS_MAX,
 328         NR_QOS_PARAMS,
 329 };
 330
 331 /* io.cost.model controls */
 332 enum {
 333         COST_CTRL,
 334         COST_MODEL,
 335         NR_COST_CTRL_PARAMS,
 336 };
 337
 338 /* builtin linear cost model coefficients */
 339 enum {
 340         I_LCOEF_RBPS,
 341         I_LCOEF_RSEQIOPS,
 342         I_LCOEF_RRANDIOPS,
 343         I_LCOEF_WBPS,
 344         I_LCOEF_WSEQIOPS,
 345         I_LCOEF_WRANDIOPS,
 346         NR_I_LCOEFS,
 347 };
 348
 349 enum {
 350         LCOEF_RPAGE,
 351         LCOEF_RSEQIO,
 352         LCOEF_RRANDIO,
 353         LCOEF_WPAGE,
 354         LCOEF_WSEQIO,
 355         LCOEF_WRANDIO,
 356         NR_LCOEFS,
 357 };
 358
 359 enum {
 360         AUTOP_INVALID,
 361         AUTOP_HDD,
 362         AUTOP_SSD_QD1,
 363         AUTOP_SSD_DFL,
 364         AUTOP_SSD_FAST,
 365 };
 366
 367 struct ioc_gq;
 368
 369 struct ioc_params {
 370         u32                             qos[NR_QOS_PARAMS];
 371         u64                             i_lcoefs[NR_I_LCOEFS];
 372         u64                             lcoefs[NR_LCOEFS];
 373         u32                             too_fast_vrate_pct;
 374         u32                             too_slow_vrate_pct;
 375 };
 376
 377 struct ioc_missed {
 378         u32                             nr_met;
 379         u32                             nr_missed;
 380         u32                             last_met;
 381         u32                             last_missed;
 382 };
 383
 384 struct ioc_pcpu_stat {
 385         struct ioc_missed               missed[2];
 386
 387         u64                             rq_wait_ns;
 388         u64                             last_rq_wait_ns;
 389 };
 390
 391 /* per device */
 392 struct ioc {
 393         struct rq_qos                   rqos;
 394
 395         bool                            enabled;
 396
 397         struct ioc_params               params;
 398         u32                             period_us;
 399         u32                             margin_us;
 400         u64                             vrate_min;
 401         u64                             vrate_max;
 402
 403         spinlock_t                      lock;
 404         struct timer_list               timer;
 405         struct list_head                active_iocgs;   /* active cgroups */
 406         struct ioc_pcpu_stat __percpu   *pcpu_stat;
 407
 408         enum ioc_running                running;
 409         atomic64_t                      vtime_rate;
 410
 411         seqcount_t                      period_seqcount;
 412         u32                             period_at;      /* wallclock starttime */
 413         u64                             period_at_vtime; /* vtime starttime */
 414
 415         atomic64_t                      cur_period;     /* inc'd each period */
 416         int                             busy_level;     /* saturation history */
 417
 418         u64                             inuse_margin_vtime;
 419         bool                            weights_updated;
 420         atomic_t                        hweight_gen;    /* for lazy hweights */
 421
 422         u64                             autop_too_fast_at;
 423         u64                             autop_too_slow_at;
 424         int                             autop_idx;
 425         bool                            user_qos_params:1;
 426         bool                            user_cost_model:1;
 427 };
 428
 429 /* per device-cgroup pair */
 430 struct ioc_gq {
 431         struct blkg_policy_data         pd;
 432         struct ioc                      *ioc;
 433
 434         /*
 435          * A iocg can get its weight from two sources - an explicit
 436          * per-device-cgroup configuration or the default weight of the
 437          * cgroup.  `cfg_weight` is the explicit per-device-cgroup
 438          * configuration.  `weight` is the effective considering both
 439          * sources.
 440          *
 441          * When an idle cgroup becomes active its `active` goes from 0 to
 442          * `weight`.  `inuse` is the surplus adjusted active weight.
 443          * `active` and `inuse` are used to calculate `hweight_active` and
 444          * `hweight_inuse`.
 445          *
 446          * `last_inuse` remembers `inuse` while an iocg is idle to persist
 447          * surplus adjustments.
 448          */
 449         u32                             cfg_weight;
 450         u32                             weight;
 451         u32                             active;
 452         u32                             inuse;
 453         u32                             last_inuse;
 454
 455         sector_t                        cursor;         /* to detect randio */
 456
 457         /*
 458          * `vtime` is this iocg's vtime cursor which progresses as IOs are
 459          * issued.  If lagging behind device vtime, the delta represents
 460          * the currently available IO budget.  If runnning ahead, the
 461          * overage.
 462          *
 463          * `vtime_done` is the same but progressed on completion rather
 464          * than issue.  The delta behind `vtime` represents the cost of
 465          * currently in-flight IOs.
 466          *
 467          * `last_vtime` is used to remember `vtime` at the end of the last
 468          * period to calculate utilization.
 469          */
 470         atomic64_t                      vtime;
 471         atomic64_t                      done_vtime;
 472         u64                             abs_vdebt;
 473         u64                             last_vtime;
 474
 475         /*
 476          * The period this iocg was last active in.  Used for deactivation
 477          * and invalidating `vtime`.
 478          */
 479         atomic64_t                      active_period;
 480         struct list_head                active_list;
 481
 482         /* see __propagate_active_weight() and current_hweight() for details */
 483         u64                             child_active_sum;
 484         u64                             child_inuse_sum;
 485         int                             hweight_gen;
 486         u32                             hweight_active;
 487         u32                             hweight_inuse;
 488         bool                            has_surplus;
 489
 490         struct wait_queue_head          waitq;
 491         struct hrtimer                  waitq_timer;
 492         struct hrtimer                  delay_timer;
 493
 494         /* usage is recorded as fractions of HWEIGHT_WHOLE */
 495         int                             usage_idx;
 496         u32                             usages[NR_USAGE_SLOTS];
 497
 498         /* this iocg's depth in the hierarchy and ancestors including self */
 499         int                             level;
 500         struct ioc_gq                   *ancestors[];
 501 };
 502
 503 /* per cgroup */
 504 struct ioc_cgrp {
 505         struct blkcg_policy_data        cpd;
 506         unsigned int                    dfl_weight;
 507 };
 508
 509 struct ioc_now {
 510         u64                             now_ns;
 511         u32                             now;
 512         u64                             vnow;
 513         u64                             vrate;
 514 };
 515
 516 struct iocg_wait {
 517         struct wait_queue_entry         wait;
 518         struct bio                      *bio;
 519         u64                             abs_cost;
 520         bool                            committed;
 521 };
 522
 523 struct iocg_wake_ctx {
 524         struct ioc_gq                   *iocg;
 525         u32                             hw_inuse;
 526         s64                             vbudget;
 527 };
 528
 529 static const struct ioc_params autop[] = {
 530         [AUTOP_HDD] = {
 531                 .qos                            = {
 532                         [QOS_RLAT]              =        250000, /* 250ms */
 533                         [QOS_WLAT]              =        250000,
 534                         [QOS_MIN]               = VRATE_MIN_PPM,
 535                         [QOS_MAX]               = VRATE_MAX_PPM,
 536                 },
 537                 .i_lcoefs                       = {
 538                         [I_LCOEF_RBPS]          =     174019176,
 539                         [I_LCOEF_RSEQIOPS]      =         41708,
 540                         [I_LCOEF_RRANDIOPS]     =           370,
 541                         [I_LCOEF_WBPS]          =     178075866,
 542                         [I_LCOEF_WSEQIOPS]      =         42705,
 543                         [I_LCOEF_WRANDIOPS]     =           378,
 544                 },
 545         },
 546         [AUTOP_SSD_QD1] = {
 547                 .qos                            = {
 548                         [QOS_RLAT]              =         25000, /* 25ms */
 549                         [QOS_WLAT]              =         25000,
 550                         [QOS_MIN]               = VRATE_MIN_PPM,
 551                         [QOS_MAX]               = VRATE_MAX_PPM,
 552                 },
 553                 .i_lcoefs                       = {
 554                         [I_LCOEF_RBPS]          =     245855193,
 555                         [I_LCOEF_RSEQIOPS]      =         61575,
 556                         [I_LCOEF_RRANDIOPS]     =          6946,
 557                         [I_LCOEF_WBPS]          =     141365009,
 558                         [I_LCOEF_WSEQIOPS]      =         33716,
 559                         [I_LCOEF_WRANDIOPS]     =         26796,
 560                 },
 561         },
 562         [AUTOP_SSD_DFL] = {
 563                 .qos                            = {
 564                         [QOS_RLAT]              =         25000, /* 25ms */
 565                         [QOS_WLAT]              =         25000,
 566                         [QOS_MIN]               = VRATE_MIN_PPM,
 567                         [QOS_MAX]               = VRATE_MAX_PPM,
 568                 },
 569                 .i_lcoefs                       = {
 570                         [I_LCOEF_RBPS]          =     488636629,
 571                         [I_LCOEF_RSEQIOPS]      =          8932,
 572                         [I_LCOEF_RRANDIOPS]     =          8518,
 573                         [I_LCOEF_WBPS]          =     427891549,
 574                         [I_LCOEF_WSEQIOPS]      =         28755,
 575                         [I_LCOEF_WRANDIOPS]     =         21940,
 576                 },
 577                 .too_fast_vrate_pct             =           500,
 578         },
 579         [AUTOP_SSD_FAST] = {
 580                 .qos                            = {
 581                         [QOS_RLAT]              =          5000, /* 5ms */
 582                         [QOS_WLAT]              =          5000,
 583                         [QOS_MIN]               = VRATE_MIN_PPM,
 584                         [QOS_MAX]               = VRATE_MAX_PPM,
 585                 },
 586                 .i_lcoefs                       = {
 587                         [I_LCOEF_RBPS]          =    3102524156LLU,
 588                         [I_LCOEF_RSEQIOPS]      =        724816,
 589                         [I_LCOEF_RRANDIOPS]     =        778122,
 590                         [I_LCOEF_WBPS]          =    1742780862LLU,
 591                         [I_LCOEF_WSEQIOPS]      =        425702,
 592                         [I_LCOEF_WRANDIOPS]     =        443193,
 593                 },
 594                 .too_slow_vrate_pct             =            10,
 595         },
 596 };
 597
 598 /*
 599  * vrate adjust percentages indexed by ioc->busy_level.  We adjust up on
 600  * vtime credit shortage and down on device saturation.
 601  */
 602 static u32 vrate_adj_pct[] =
 603         { 0, 0, 0, 0,
 604           1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
 605           2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 606           4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 8, 8, 8, 16 };
 607
 608 static struct blkcg_policy blkcg_policy_iocost;
 609
 610 /* accessors and helpers */
 611 static struct ioc *rqos_to_ioc(struct rq_qos *rqos)
 612 {
 613         return container_of(rqos, struct ioc, rqos);
 614 }
 615
 616 static struct ioc *q_to_ioc(struct request_queue *q)
 617 {
 618         return rqos_to_ioc(rq_qos_id(q, RQ_QOS_COST));
 619 }
 620
 621 static const char *q_name(struct request_queue *q)
 622 {
 623         if (test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
 624                 return kobject_name(q->kobj.parent);
 625         else
 626                 return "<unknown>";
 627 }
 628
 629 static const char __maybe_unused *ioc_name(struct ioc *ioc)
 630 {
 631         return q_name(ioc->rqos.q);
 632 }
 633
 634 static struct ioc_gq *pd_to_iocg(struct blkg_policy_data *pd)
 635 {
 636         return pd ? container_of(pd, struct ioc_gq, pd) : NULL;
 637 }
 638
 639 static struct ioc_gq *blkg_to_iocg(struct blkcg_gq *blkg)
 640 {
 641         return pd_to_iocg(blkg_to_pd(blkg, &blkcg_policy_iocost));
 642 }
 643
 644 static struct blkcg_gq *iocg_to_blkg(struct ioc_gq *iocg)
 645 {
 646         return pd_to_blkg(&iocg->pd);
 647 }
 648
 649 static struct ioc_cgrp *blkcg_to_iocc(struct blkcg *blkcg)
 650 {
 651         return container_of(blkcg_to_cpd(blkcg, &blkcg_policy_iocost),
 652                             struct ioc_cgrp, cpd);
 653 }
 654
 655 /*
 656  * Scale @abs_cost to the inverse of @hw_inuse.  The lower the hierarchical
 657  * weight, the more expensive each IO.  Must round up.
 658  */
 659 static u64 abs_cost_to_cost(u64 abs_cost, u32 hw_inuse)
 660 {
 661         return DIV64_U64_ROUND_UP(abs_cost * HWEIGHT_WHOLE, hw_inuse);
 662 }
 663
 664 /*
 665  * The inverse of abs_cost_to_cost().  Must round up.
 666  */
 667 static u64 cost_to_abs_cost(u64 cost, u32 hw_inuse)
 668 {
 669         return DIV64_U64_ROUND_UP(cost * hw_inuse, HWEIGHT_WHOLE);
 670 }
 671
 672 static void iocg_commit_bio(struct ioc_gq *iocg, struct bio *bio, u64 cost)
 673 {
 674         bio->bi_iocost_cost = cost;
 675         atomic64_add(cost, &iocg->vtime);
 676 }
 677
 678 #define CREATE_TRACE_POINTS
 679 #include <trace/events/iocost.h>
 680
 681 /* latency Qos params changed, update period_us and all the dependent params */
 682 static void ioc_refresh_period_us(struct ioc *ioc)
 683 {
 684         u32 ppm, lat, multi, period_us;
 685
 686         lockdep_assert_held(&ioc->lock);
 687
 688         /* pick the higher latency target */
 689         if (ioc->params.qos[QOS_RLAT] >= ioc->params.qos[QOS_WLAT]) {
 690                 ppm = ioc->params.qos[QOS_RPPM];
 691                 lat = ioc->params.qos[QOS_RLAT];
 692         } else {
 693                 ppm = ioc->params.qos[QOS_WPPM];
 694                 lat = ioc->params.qos[QOS_WLAT];
 695         }
 696
 697         /*
 698          * We want the period to be long enough to contain a healthy number
 699          * of IOs while short enough for granular control.  Define it as a
 700          * multiple of the latency target.  Ideally, the multiplier should
 701          * be scaled according to the percentile so that it would nominally
 702          * contain a certain number of requests.  Let's be simpler and
 703          * scale it linearly so that it's 2x >= pct(90) and 10x at pct(50).
 704          */
 705         if (ppm)
 706                 multi = max_t(u32, (MILLION - ppm) / 50000, 2);
 707         else
 708                 multi = 2;
 709         period_us = multi * lat;
 710         period_us = clamp_t(u32, period_us, MIN_PERIOD, MAX_PERIOD);
 711
 712         /* calculate dependent params */
 713         ioc->period_us = period_us;
 714         ioc->margin_us = period_us * MARGIN_PCT / 100;
 715         ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(
 716                         period_us * VTIME_PER_USEC * INUSE_MARGIN_PCT, 100);
 717 }
 718
 719 static int ioc_autop_idx(struct ioc *ioc)
 720 {
 721         int idx = ioc->autop_idx;
 722         const struct ioc_params *p = &autop[idx];
 723         u32 vrate_pct;
 724         u64 now_ns;
 725
 726         /* rotational? */
 727         if (!blk_queue_nonrot(ioc->rqos.q))
 728                 return AUTOP_HDD;
 729
 730         /* handle SATA SSDs w/ broken NCQ */
 731         if (blk_queue_depth(ioc->rqos.q) == 1)
 732                 return AUTOP_SSD_QD1;
 733
 734         /* use one of the normal ssd sets */
 735         if (idx < AUTOP_SSD_DFL)
 736                 return AUTOP_SSD_DFL;
 737
 738         /* if user is overriding anything, maintain what was there */
 739         if (ioc->user_qos_params || ioc->user_cost_model)
 740                 return idx;
 741
 742         /* step up/down based on the vrate */
 743         vrate_pct = div64_u64(atomic64_read(&ioc->vtime_rate) * 100,
 744                               VTIME_PER_USEC);
 745         now_ns = ktime_get_ns();
 746
 747         if (p->too_fast_vrate_pct && p->too_fast_vrate_pct <= vrate_pct) {
 748                 if (!ioc->autop_too_fast_at)
 749                         ioc->autop_too_fast_at = now_ns;
 750                 if (now_ns - ioc->autop_too_fast_at >= AUTOP_CYCLE_NSEC)
 751                         return idx + 1;
 752         } else {
 753                 ioc->autop_too_fast_at = 0;
 754         }
 755
 756         if (p->too_slow_vrate_pct && p->too_slow_vrate_pct >= vrate_pct) {
 757                 if (!ioc->autop_too_slow_at)
 758                         ioc->autop_too_slow_at = now_ns;
 759                 if (now_ns - ioc->autop_too_slow_at >= AUTOP_CYCLE_NSEC)
 760                         return idx - 1;
 761         } else {
 762                 ioc->autop_too_slow_at = 0;
 763         }
 764
 765         return idx;
 766 }
 767
 768 /*
 769  * Take the followings as input
 770  *
 771  *  @bps        maximum sequential throughput
 772  *  @seqiops    maximum sequential 4k iops
 773  *  @randiops   maximum random 4k iops
 774  *
 775  * and calculate the linear model cost coefficients.
 776  *
 777  *  *@page      per-page cost           1s / (@bps / 4096)
 778  *  *@seqio     base cost of a seq IO   max((1s / @seqiops) - *@page, 0)
 779  *  @randiops   base cost of a rand IO  max((1s / @randiops) - *@page, 0)
 780  */
 781 static void calc_lcoefs(u64 bps, u64 seqiops, u64 randiops,
 782                         u64 *page, u64 *seqio, u64 *randio)
 783 {
 784         u64 v;
 785
 786         *page = *seqio = *randio = 0;
 787
 788         if (bps) {
 789                 u64 bps_pages = DIV_ROUND_UP_ULL(bps, IOC_PAGE_SIZE);
 790
 791                 if (bps_pages)
 792                         *page = DIV64_U64_ROUND_UP(VTIME_PER_SEC, bps_pages);
 793                 else
 794                         *page = 1;
 795         }
 796
 797         if (seqiops) {
 798                 v = DIV64_U64_ROUND_UP(VTIME_PER_SEC, seqiops);
 799                 if (v > *page)
 800                         *seqio = v - *page;
 801         }
 802
 803         if (randiops) {
 804                 v = DIV64_U64_ROUND_UP(VTIME_PER_SEC, randiops);
 805                 if (v > *page)
 806                         *randio = v - *page;
 807         }
 808 }
 809
 810 static void ioc_refresh_lcoefs(struct ioc *ioc)
 811 {
 812         u64 *u = ioc->params.i_lcoefs;
 813         u64 *c = ioc->params.lcoefs;
 814
 815         calc_lcoefs(u[I_LCOEF_RBPS], u[I_LCOEF_RSEQIOPS], u[I_LCOEF_RRANDIOPS],
 816                     &c[LCOEF_RPAGE], &c[LCOEF_RSEQIO], &c[LCOEF_RRANDIO]);
 817         calc_lcoefs(u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS],
 818                     &c[LCOEF_WPAGE], &c[LCOEF_WSEQIO], &c[LCOEF_WRANDIO]);
 819 }
 820
 821 static bool ioc_refresh_params(struct ioc *ioc, bool force)
 822 {
 823         const struct ioc_params *p;
 824         int idx;
 825
 826         lockdep_assert_held(&ioc->lock);
 827
 828         idx = ioc_autop_idx(ioc);
 829         p = &autop[idx];
 830
 831         if (idx == ioc->autop_idx && !force)
 832                 return false;
 833
 834         if (idx != ioc->autop_idx)
 835                 atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);
 836
 837         ioc->autop_idx = idx;
 838         ioc->autop_too_fast_at = 0;
 839         ioc->autop_too_slow_at = 0;
 840
 841         if (!ioc->user_qos_params)
 842                 memcpy(ioc->params.qos, p->qos, sizeof(p->qos));
 843         if (!ioc->user_cost_model)
 844                 memcpy(ioc->params.i_lcoefs, p->i_lcoefs, sizeof(p->i_lcoefs));
 845
 846         ioc_refresh_period_us(ioc);
 847         ioc_refresh_lcoefs(ioc);
 848
 849         ioc->vrate_min = DIV64_U64_ROUND_UP((u64)ioc->params.qos[QOS_MIN] *
 850                                             VTIME_PER_USEC, MILLION);
 851         ioc->vrate_max = div64_u64((u64)ioc->params.qos[QOS_MAX] *
 852                                    VTIME_PER_USEC, MILLION);
 853
 854         return true;
 855 }
 856
 857 /* take a snapshot of the current [v]time and vrate */
 858 static void ioc_now(struct ioc *ioc, struct ioc_now *now)
 859 {
 860         unsigned seq;
 861
 862         now->now_ns = ktime_get();
 863         now->now = ktime_to_us(now->now_ns);
 864         now->vrate = atomic64_read(&ioc->vtime_rate);
 865
 866         /*
 867          * The current vtime is
 868          *
 869          *   vtime at period start + (wallclock time since the start) * vrate
 870          *
 871          * As a consistent snapshot of `period_at_vtime` and `period_at` is
 872          * needed, they're seqcount protected.
 873          */
 874         do {
 875                 seq = read_seqcount_begin(&ioc->period_seqcount);
 876                 now->vnow = ioc->period_at_vtime +
 877                         (now->now - ioc->period_at) * now->vrate;
 878         } while (read_seqcount_retry(&ioc->period_seqcount, seq));
 879 }
 880
 881 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now)
 882 {
 883         lockdep_assert_held(&ioc->lock);
 884         WARN_ON_ONCE(ioc->running != IOC_RUNNING);
 885
 886         write_seqcount_begin(&ioc->period_seqcount);
 887         ioc->period_at = now->now;
 888         ioc->period_at_vtime = now->vnow;
 889         write_seqcount_end(&ioc->period_seqcount);
 890
 891         ioc->timer.expires = jiffies + usecs_to_jiffies(ioc->period_us);
 892         add_timer(&ioc->timer);
 893 }
 894
 895 /*
 896  * Update @iocg's `active` and `inuse` to @active and @inuse, update level
 897  * weight sums and propagate upwards accordingly.
 898  */
 899 static void __propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse)
 900 {
 901         struct ioc *ioc = iocg->ioc;
 902         int lvl;
 903
 904         lockdep_assert_held(&ioc->lock);
 905
 906         inuse = min(active, inuse);
 907
 908         for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
 909                 struct ioc_gq *parent = iocg->ancestors[lvl];
 910                 struct ioc_gq *child = iocg->ancestors[lvl + 1];
 911                 u32 parent_active = 0, parent_inuse = 0;
 912
 913                 /* update the level sums */
 914                 parent->child_active_sum += (s32)(active - child->active);
 915                 parent->child_inuse_sum += (s32)(inuse - child->inuse);
 916                 /* apply the udpates */
 917                 child->active = active;
 918                 child->inuse = inuse;
 919
 920                 /*
 921                  * The delta between inuse and active sums indicates that
 922                  * that much of weight is being given away.  Parent's inuse
 923                  * and active should reflect the ratio.
 924                  */
 925                 if (parent->child_active_sum) {
 926                         parent_active = parent->weight;
 927                         parent_inuse = DIV64_U64_ROUND_UP(
 928                                 parent_active * parent->child_inuse_sum,
 929                                 parent->child_active_sum);
 930                 }
 931
 932                 /* do we need to keep walking up? */
 933                 if (parent_active == parent->active &&
 934                     parent_inuse == parent->inuse)
 935                         break;
 936
 937                 active = parent_active;
 938                 inuse = parent_inuse;
 939         }
 940
 941         ioc->weights_updated = true;
 942 }
 943
 944 static void commit_active_weights(struct ioc *ioc)
 945 {
 946         lockdep_assert_held(&ioc->lock);
 947
 948         if (ioc->weights_updated) {
 949                 /* paired with rmb in current_hweight(), see there */
 950                 smp_wmb();
 951                 atomic_inc(&ioc->hweight_gen);
 952                 ioc->weights_updated = false;
 953         }
 954 }
 955
 956 static void propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse)
 957 {
 958         __propagate_active_weight(iocg, active, inuse);
 959         commit_active_weights(iocg->ioc);
 960 }
 961
 962 static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep)
 963 {
 964         struct ioc *ioc = iocg->ioc;
 965         int lvl;
 966         u32 hwa, hwi;
 967         int ioc_gen;
 968
 969         /* hot path - if uptodate, use cached */
 970         ioc_gen = atomic_read(&ioc->hweight_gen);
 971         if (ioc_gen == iocg->hweight_gen)
 972                 goto out;
 973
 974         /*
 975          * Paired with wmb in commit_active_weights().  If we saw the
 976          * updated hweight_gen, all the weight updates from
 977          * __propagate_active_weight() are visible too.
 978          *
 979          * We can race with weight updates during calculation and get it
 980          * wrong.  However, hweight_gen would have changed and a future
 981          * reader will recalculate and we're guaranteed to discard the
 982          * wrong result soon.
 983          */
 984         smp_rmb();
 985
 986         hwa = hwi = HWEIGHT_WHOLE;
 987         for (lvl = 0; lvl <= iocg->level - 1; lvl++) {
 988                 struct ioc_gq *parent = iocg->ancestors[lvl];
 989                 struct ioc_gq *child = iocg->ancestors[lvl + 1];
 990                 u32 active_sum = READ_ONCE(parent->child_active_sum);
 991                 u32 inuse_sum = READ_ONCE(parent->child_inuse_sum);
 992                 u32 active = READ_ONCE(child->active);
 993                 u32 inuse = READ_ONCE(child->inuse);
 994
 995                 /* we can race with deactivations and either may read as zero */
 996                 if (!active_sum || !inuse_sum)
 997                         continue;
 998
 999                 active_sum = max(active, active_sum);
1000                 hwa = hwa * active / active_sum;        /* max 16bits * 10000 */
1001
1002                 inuse_sum = max(inuse, inuse_sum);
1003                 hwi = hwi * inuse / inuse_sum;          /* max 16bits * 10000 */
1004         }
1005
1006         iocg->hweight_active = max_t(u32, hwa, 1);
1007         iocg->hweight_inuse = max_t(u32, hwi, 1);
1008         iocg->hweight_gen = ioc_gen;
1009 out:
1010         if (hw_activep)
1011                 *hw_activep = iocg->hweight_active;
1012         if (hw_inusep)
1013                 *hw_inusep = iocg->hweight_inuse;
1014 }
1015
1016 static void weight_updated(struct ioc_gq *iocg)
1017 {
1018         struct ioc *ioc = iocg->ioc;
1019         struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1020         struct ioc_cgrp *iocc = blkcg_to_iocc(blkg->blkcg);
1021         u32 weight;
1022
1023         lockdep_assert_held(&ioc->lock);
1024
1025         weight = iocg->cfg_weight ?: iocc->dfl_weight;
1026         if (weight != iocg->weight && iocg->active)
1027                 propagate_active_weight(iocg, weight,
1028                         DIV64_U64_ROUND_UP(iocg->inuse * weight, iocg->weight));
1029         iocg->weight = weight;
1030 }
1031
1032 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
1033 {
1034         struct ioc *ioc = iocg->ioc;
1035         u64 last_period, cur_period, max_period_delta;
1036         u64 vtime, vmargin, vmin;
1037         int i;
1038
1039         /*
1040          * If seem to be already active, just update the stamp to tell the
1041          * timer that we're still active.  We don't mind occassional races.
1042          */
1043         if (!list_empty(&iocg->active_list)) {
1044                 ioc_now(ioc, now);
1045                 cur_period = atomic64_read(&ioc->cur_period);
1046                 if (atomic64_read(&iocg->active_period) != cur_period)
1047                         atomic64_set(&iocg->active_period, cur_period);
1048                 return true;
1049         }
1050
1051         /* racy check on internal node IOs, treat as root level IOs */
1052         if (iocg->child_active_sum)
1053                 return false;
1054
1055         spin_lock_irq(&ioc->lock);
1056
1057         ioc_now(ioc, now);
1058
1059         /* update period */
1060         cur_period = atomic64_read(&ioc->cur_period);
1061         last_period = atomic64_read(&iocg->active_period);
1062         atomic64_set(&iocg->active_period, cur_period);
1063
1064         /* already activated or breaking leaf-only constraint? */
1065         if (!list_empty(&iocg->active_list))
1066                 goto succeed_unlock;
1067         for (i = iocg->level - 1; i > 0; i--)
1068                 if (!list_empty(&iocg->ancestors[i]->active_list))
1069                         goto fail_unlock;
1070
1071         if (iocg->child_active_sum)
1072                 goto fail_unlock;
1073
1074         /*
1075          * vtime may wrap when vrate is raised substantially due to
1076          * underestimated IO costs.  Look at the period and ignore its
1077          * vtime if the iocg has been idle for too long.  Also, cap the
1078          * budget it can start with to the margin.
1079          */
1080         max_period_delta = DIV64_U64_ROUND_UP(VTIME_VALID_DUR, ioc->period_us);
1081         vtime = atomic64_read(&iocg->vtime);
1082         vmargin = ioc->margin_us * now->vrate;
1083         vmin = now->vnow - vmargin;
1084
1085         if (last_period + max_period_delta < cur_period ||
1086             time_before64(vtime, vmin)) {
1087                 atomic64_add(vmin - vtime, &iocg->vtime);
1088                 atomic64_add(vmin - vtime, &iocg->done_vtime);
1089                 vtime = vmin;
1090         }
1091
1092         /*
1093          * Activate, propagate weight and start period timer if not
1094          * running.  Reset hweight_gen to avoid accidental match from
1095          * wrapping.
1096          */
1097         iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;
1098         list_add(&iocg->active_list, &ioc->active_iocgs);
1099         propagate_active_weight(iocg, iocg->weight,
1100                                 iocg->last_inuse ?: iocg->weight);
1101
1102         TRACE_IOCG_PATH(iocg_activate, iocg, now,
1103                         last_period, cur_period, vtime);
1104
1105         iocg->last_vtime = vtime;
1106
1107         if (ioc->running == IOC_IDLE) {
1108                 ioc->running = IOC_RUNNING;
1109                 ioc_start_period(ioc, now);
1110         }
1111
1112 succeed_unlock:
1113         spin_unlock_irq(&ioc->lock);
1114         return true;
1115
1116 fail_unlock:
1117         spin_unlock_irq(&ioc->lock);
1118         return false;
1119 }
1120
1121 static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode,
1122                         int flags, void *key)
1123 {
1124         struct iocg_wait *wait = container_of(wq_entry, struct iocg_wait, wait);
1125         struct iocg_wake_ctx *ctx = (struct iocg_wake_ctx *)key;
1126         u64 cost = abs_cost_to_cost(wait->abs_cost, ctx->hw_inuse);
1127
1128         ctx->vbudget -= cost;
1129
1130         if (ctx->vbudget < 0)
1131                 return -1;
1132
1133         iocg_commit_bio(ctx->iocg, wait->bio, cost);
1134
1135         /*
1136          * autoremove_wake_function() removes the wait entry only when it
1137          * actually changed the task state.  We want the wait always
1138          * removed.  Remove explicitly and use default_wake_function().
1139          */
1140         list_del_init(&wq_entry->entry);
1141         wait->committed = true;
1142
1143         default_wake_function(wq_entry, mode, flags, key);
1144         return 0;
1145 }
1146
1147 static void iocg_kick_waitq(struct ioc_gq *iocg, struct ioc_now *now)
1148 {
1149         struct ioc *ioc = iocg->ioc;
1150         struct iocg_wake_ctx ctx = { .iocg = iocg };
1151         u64 margin_ns = (u64)(ioc->period_us *
1152                               WAITQ_TIMER_MARGIN_PCT / 100) * NSEC_PER_USEC;
1153         u64 vdebt, vshortage, expires, oexpires;
1154         s64 vbudget;
1155         u32 hw_inuse;
1156
1157         lockdep_assert_held(&iocg->waitq.lock);
1158
1159         current_hweight(iocg, NULL, &hw_inuse);
1160         vbudget = now->vnow - atomic64_read(&iocg->vtime);
1161
1162         /* pay off debt */
1163         vdebt = abs_cost_to_cost(iocg->abs_vdebt, hw_inuse);
1164         if (vdebt && vbudget > 0) {
1165                 u64 delta = min_t(u64, vbudget, vdebt);
1166                 u64 abs_delta = min(cost_to_abs_cost(delta, hw_inuse),
1167                                     iocg->abs_vdebt);
1168
1169                 atomic64_add(delta, &iocg->vtime);
1170                 atomic64_add(delta, &iocg->done_vtime);
1171                 iocg->abs_vdebt -= abs_delta;
1172         }
1173
1174         /*
1175          * Wake up the ones which are due and see how much vtime we'll need
1176          * for the next one.
1177          */
1178         ctx.hw_inuse = hw_inuse;
1179         ctx.vbudget = vbudget - vdebt;
1180         __wake_up_locked_key(&iocg->waitq, TASK_NORMAL, &ctx);
1181         if (!waitqueue_active(&iocg->waitq))
1182                 return;
1183         if (WARN_ON_ONCE(ctx.vbudget >= 0))
1184                 return;
1185
1186         /* determine next wakeup, add a quarter margin to guarantee chunking */
1187         vshortage = -ctx.vbudget;
1188         expires = now->now_ns +
1189                 DIV64_U64_ROUND_UP(vshortage, now->vrate) * NSEC_PER_USEC;
1190         expires += margin_ns / 4;
1191
1192         /* if already active and close enough, don't bother */
1193         oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->waitq_timer));
1194         if (hrtimer_is_queued(&iocg->waitq_timer) &&
1195             abs(oexpires - expires) <= margin_ns / 4)
1196                 return;
1197
1198         hrtimer_start_range_ns(&iocg->waitq_timer, ns_to_ktime(expires),
1199                                margin_ns / 4, HRTIMER_MODE_ABS);
1200 }
1201
1202 static enum hrtimer_restart iocg_waitq_timer_fn(struct hrtimer *timer)
1203 {
1204         struct ioc_gq *iocg = container_of(timer, struct ioc_gq, waitq_timer);
1205         struct ioc_now now;
1206         unsigned long flags;
1207
1208         ioc_now(iocg->ioc, &now);
1209
1210         spin_lock_irqsave(&iocg->waitq.lock, flags);
1211         iocg_kick_waitq(iocg, &now);
1212         spin_unlock_irqrestore(&iocg->waitq.lock, flags);
1213
1214         return HRTIMER_NORESTART;
1215 }
1216
1217 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now, u64 cost)
1218 {
1219         struct ioc *ioc = iocg->ioc;
1220         struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1221         u64 vtime = atomic64_read(&iocg->vtime);
1222         u64 vmargin = ioc->margin_us * now->vrate;
1223         u64 margin_ns = ioc->margin_us * NSEC_PER_USEC;
1224         u64 expires, oexpires;
1225         u32 hw_inuse;
1226
1227         lockdep_assert_held(&iocg->waitq.lock);
1228
1229         /* debt-adjust vtime */
1230         current_hweight(iocg, NULL, &hw_inuse);
1231         vtime += abs_cost_to_cost(iocg->abs_vdebt, hw_inuse);
1232
1233         /*
1234          * Clear or maintain depending on the overage. Non-zero vdebt is what
1235          * guarantees that @iocg is online and future iocg_kick_delay() will
1236          * clear use_delay. Don't leave it on when there's no vdebt.
1237          */
1238         if (!iocg->abs_vdebt || time_before_eq64(vtime, now->vnow)) {
1239                 blkcg_clear_delay(blkg);
1240                 return false;
1241         }
1242         if (!atomic_read(&blkg->use_delay) &&
1243             time_before_eq64(vtime, now->vnow + vmargin))
1244                 return false;
1245
1246         /* use delay */
1247         if (cost) {
1248                 u64 cost_ns = DIV64_U64_ROUND_UP(cost * NSEC_PER_USEC,
1249                                                  now->vrate);
1250                 blkcg_add_delay(blkg, now->now_ns, cost_ns);
1251         }
1252         blkcg_use_delay(blkg);
1253
1254         expires = now->now_ns + DIV64_U64_ROUND_UP(vtime - now->vnow,
1255                                                    now->vrate) * NSEC_PER_USEC;
1256
1257         /* if already active and close enough, don't bother */
1258         oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->delay_timer));
1259         if (hrtimer_is_queued(&iocg->delay_timer) &&
1260             abs(oexpires - expires) <= margin_ns / 4)
1261                 return true;
1262
1263         hrtimer_start_range_ns(&iocg->delay_timer, ns_to_ktime(expires),
1264                                margin_ns / 4, HRTIMER_MODE_ABS);
1265         return true;
1266 }
1267
1268 static enum hrtimer_restart iocg_delay_timer_fn(struct hrtimer *timer)
1269 {
1270         struct ioc_gq *iocg = container_of(timer, struct ioc_gq, delay_timer);
1271         struct ioc_now now;
1272         unsigned long flags;
1273
1274         spin_lock_irqsave(&iocg->waitq.lock, flags);
1275         ioc_now(iocg->ioc, &now);
1276         iocg_kick_delay(iocg, &now, 0);
1277         spin_unlock_irqrestore(&iocg->waitq.lock, flags);
1278
1279         return HRTIMER_NORESTART;
1280 }
1281
1282 static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p)
1283 {
1284         u32 nr_met[2] = { };
1285         u32 nr_missed[2] = { };
1286         u64 rq_wait_ns = 0;
1287         int cpu, rw;
1288
1289         for_each_online_cpu(cpu) {
1290                 struct ioc_pcpu_stat *stat = per_cpu_ptr(ioc->pcpu_stat, cpu);
1291                 u64 this_rq_wait_ns;
1292
1293                 for (rw = READ; rw <= WRITE; rw++) {
1294                         u32 this_met = READ_ONCE(stat->missed[rw].nr_met);
1295                         u32 this_missed = READ_ONCE(stat->missed[rw].nr_missed);
1296
1297                         nr_met[rw] += this_met - stat->missed[rw].last_met;
1298                         nr_missed[rw] += this_missed - stat->missed[rw].last_missed;
1299                         stat->missed[rw].last_met = this_met;
1300                         stat->missed[rw].last_missed = this_missed;
1301                 }
1302
1303                 this_rq_wait_ns = READ_ONCE(stat->rq_wait_ns);
1304                 rq_wait_ns += this_rq_wait_ns - stat->last_rq_wait_ns;
1305                 stat->last_rq_wait_ns = this_rq_wait_ns;
1306         }
1307
1308         for (rw = READ; rw <= WRITE; rw++) {
1309                 if (nr_met[rw] + nr_missed[rw])
1310                         missed_ppm_ar[rw] =
1311                                 DIV64_U64_ROUND_UP((u64)nr_missed[rw] * MILLION,
1312                                                    nr_met[rw] + nr_missed[rw]);
1313                 else
1314                         missed_ppm_ar[rw] = 0;
1315         }
1316
1317         *rq_wait_pct_p = div64_u64(rq_wait_ns * 100,
1318                                    ioc->period_us * NSEC_PER_USEC);
1319 }
1320
1321 /* was iocg idle this period? */
1322 static bool iocg_is_idle(struct ioc_gq *iocg)
1323 {
1324         struct ioc *ioc = iocg->ioc;
1325
1326         /* did something get issued this period? */
1327         if (atomic64_read(&iocg->active_period) ==
1328             atomic64_read(&ioc->cur_period))
1329                 return false;
1330
1331         /* is something in flight? */
1332         if (atomic64_read(&iocg->done_vtime) != atomic64_read(&iocg->vtime))
1333                 return false;
1334
1335         return true;
1336 }
1337
1338 /* returns usage with margin added if surplus is large enough */
1339 static u32 surplus_adjusted_hweight_inuse(u32 usage, u32 hw_inuse)
1340 {
1341         /* add margin */
1342         usage = DIV_ROUND_UP(usage * SURPLUS_SCALE_PCT, 100);
1343         usage += SURPLUS_SCALE_ABS;
1344
1345         /* don't bother if the surplus is too small */
1346         if (usage + SURPLUS_MIN_ADJ_DELTA > hw_inuse)
1347                 return 0;
1348
1349         return usage;
1350 }
1351
1352 static void ioc_timer_fn(struct timer_list *timer)
1353 {
1354         struct ioc *ioc = container_of(timer, struct ioc, timer);
1355         struct ioc_gq *iocg, *tiocg;
1356         struct ioc_now now;
1357         int nr_surpluses = 0, nr_shortages = 0, nr_lagging = 0;
1358         u32 ppm_rthr = MILLION - ioc->params.qos[QOS_RPPM];
1359         u32 ppm_wthr = MILLION - ioc->params.qos[QOS_WPPM];
1360         u32 missed_ppm[2], rq_wait_pct;
1361         u64 period_vtime;
1362         int prev_busy_level, i;
1363
1364         /* how were the latencies during the period? */
1365         ioc_lat_stat(ioc, missed_ppm, &rq_wait_pct);
1366
1367         /* take care of active iocgs */
1368         spin_lock_irq(&ioc->lock);
1369
1370         ioc_now(ioc, &now);
1371
1372         period_vtime = now.vnow - ioc->period_at_vtime;
1373         if (WARN_ON_ONCE(!period_vtime)) {
1374                 spin_unlock_irq(&ioc->lock);
1375                 return;
1376         }
1377
1378         /*
1379          * Waiters determine the sleep durations based on the vrate they
1380          * saw at the time of sleep.  If vrate has increased, some waiters
1381          * could be sleeping for too long.  Wake up tardy waiters which
1382          * should have woken up in the last period and expire idle iocgs.
1383          */
1384         list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {
1385                 if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
1386                     !iocg_is_idle(iocg))
1387                         continue;
1388
1389                 spin_lock(&iocg->waitq.lock);
1390
1391                 if (waitqueue_active(&iocg->waitq) || iocg->abs_vdebt) {
1392                         /* might be oversleeping vtime / hweight changes, kick */
1393                         iocg_kick_waitq(iocg, &now);
1394                         iocg_kick_delay(iocg, &now, 0);
1395                 } else if (iocg_is_idle(iocg)) {
1396                         /* no waiter and idle, deactivate */
1397                         iocg->last_inuse = iocg->inuse;
1398                         __propagate_active_weight(iocg, 0, 0);
1399                         list_del_init(&iocg->active_list);
1400                 }
1401
1402                 spin_unlock(&iocg->waitq.lock);
1403         }
1404         commit_active_weights(ioc);
1405
1406         /* calc usages and see whether some weights need to be moved around */
1407         list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
1408                 u64 vdone, vtime, vusage, vmargin, vmin;
1409                 u32 hw_active, hw_inuse, usage;
1410
1411                 /*
1412                  * Collect unused and wind vtime closer to vnow to prevent
1413                  * iocgs from accumulating a large amount of budget.
1414                  */
1415                 vdone = atomic64_read(&iocg->done_vtime);
1416                 vtime = atomic64_read(&iocg->vtime);
1417                 current_hweight(iocg, &hw_active, &hw_inuse);
1418
1419                 /*
1420                  * Latency QoS detection doesn't account for IOs which are
1421                  * in-flight for longer than a period.  Detect them by
1422                  * comparing vdone against period start.  If lagging behind
1423                  * IOs from past periods, don't increase vrate.
1424                  */
1425                 if ((ppm_rthr != MILLION || ppm_wthr != MILLION) &&
1426                     !atomic_read(&iocg_to_blkg(iocg)->use_delay) &&
1427                     time_after64(vtime, vdone) &&
1428                     time_after64(vtime, now.vnow -
1429                                  MAX_LAGGING_PERIODS * period_vtime) &&
1430                     time_before64(vdone, now.vnow - period_vtime))
1431                         nr_lagging++;
1432
1433                 if (waitqueue_active(&iocg->waitq))
1434                         vusage = now.vnow - iocg->last_vtime;
1435                 else if (time_before64(iocg->last_vtime, vtime))
1436                         vusage = vtime - iocg->last_vtime;
1437                 else
1438                         vusage = 0;
1439
1440                 iocg->last_vtime += vusage;
1441                 /*
1442                  * Factor in in-flight vtime into vusage to avoid
1443                  * high-latency completions appearing as idle.  This should
1444                  * be done after the above ->last_time adjustment.
1445                  */
1446                 vusage = max(vusage, vtime - vdone);
1447
1448                 /* calculate hweight based usage ratio and record */
1449                 if (vusage) {
1450                         usage = DIV64_U64_ROUND_UP(vusage * hw_inuse,
1451                                                    period_vtime);
1452                         iocg->usage_idx = (iocg->usage_idx + 1) % NR_USAGE_SLOTS;
1453                         iocg->usages[iocg->usage_idx] = usage;
1454                 } else {
1455                         usage = 0;
1456                 }
1457
1458                 /* see whether there's surplus vtime */
1459                 vmargin = ioc->margin_us * now.vrate;
1460                 vmin = now.vnow - vmargin;
1461
1462                 iocg->has_surplus = false;
1463
1464                 if (!waitqueue_active(&iocg->waitq) &&
1465                     time_before64(vtime, vmin)) {
1466                         u64 delta = vmin - vtime;
1467
1468                         /* throw away surplus vtime */
1469                         atomic64_add(delta, &iocg->vtime);
1470                         atomic64_add(delta, &iocg->done_vtime);
1471                         iocg->last_vtime += delta;
1472                         /* if usage is sufficiently low, maybe it can donate */
1473                         if (surplus_adjusted_hweight_inuse(usage, hw_inuse)) {
1474                                 iocg->has_surplus = true;
1475                                 nr_surpluses++;
1476                         }
1477                 } else if (hw_inuse < hw_active) {
1478                         u32 new_hwi, new_inuse;
1479
1480                         /* was donating but might need to take back some */
1481                         if (waitqueue_active(&iocg->waitq)) {
1482                                 new_hwi = hw_active;
1483                         } else {
1484                                 new_hwi = max(hw_inuse,
1485                                               usage * SURPLUS_SCALE_PCT / 100 +
1486                                               SURPLUS_SCALE_ABS);
1487                         }
1488
1489                         new_inuse = div64_u64((u64)iocg->inuse * new_hwi,
1490                                               hw_inuse);
1491                         new_inuse = clamp_t(u32, new_inuse, 1, iocg->active);
1492
1493                         if (new_inuse > iocg->inuse) {
1494                                 TRACE_IOCG_PATH(inuse_takeback, iocg, &now,
1495                                                 iocg->inuse, new_inuse,
1496                                                 hw_inuse, new_hwi);
1497                                 __propagate_active_weight(iocg, iocg->weight,
1498                                                           new_inuse);
1499                         }
1500                 } else {
1501                         /* genuninely out of vtime */
1502                         nr_shortages++;
1503                 }
1504         }
1505
1506         if (!nr_shortages || !nr_surpluses)
1507                 goto skip_surplus_transfers;
1508
1509         /* there are both shortages and surpluses, transfer surpluses */
1510         list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
1511                 u32 usage, hw_active, hw_inuse, new_hwi, new_inuse;
1512                 int nr_valid = 0;
1513
1514                 if (!iocg->has_surplus)
1515                         continue;
1516
1517                 /* base the decision on max historical usage */
1518                 for (i = 0, usage = 0; i < NR_USAGE_SLOTS; i++) {
1519                         if (iocg->usages[i]) {
1520                                 usage = max(usage, iocg->usages[i]);
1521                                 nr_valid++;
1522                         }
1523                 }
1524                 if (nr_valid < MIN_VALID_USAGES)
1525                         continue;
1526
1527                 current_hweight(iocg, &hw_active, &hw_inuse);
1528                 new_hwi = surplus_adjusted_hweight_inuse(usage, hw_inuse);
1529                 if (!new_hwi)
1530                         continue;
1531
1532                 new_inuse = DIV64_U64_ROUND_UP((u64)iocg->inuse * new_hwi,
1533                                                hw_inuse);
1534                 if (new_inuse < iocg->inuse) {
1535                         TRACE_IOCG_PATH(inuse_giveaway, iocg, &now,
1536                                         iocg->inuse, new_inuse,
1537                                         hw_inuse, new_hwi);
1538                         __propagate_active_weight(iocg, iocg->weight, new_inuse);
1539                 }
1540         }
1541 skip_surplus_transfers:
1542         commit_active_weights(ioc);
1543
1544         /*
1545          * If q is getting clogged or we're missing too much, we're issuing
1546          * too much IO and should lower vtime rate.  If we're not missing
1547          * and experiencing shortages but not surpluses, we're too stingy
1548          * and should increase vtime rate.
1549          */
1550         prev_busy_level = ioc->busy_level;
1551         if (rq_wait_pct > RQ_WAIT_BUSY_PCT ||
1552             missed_ppm[READ] > ppm_rthr ||
1553             missed_ppm[WRITE] > ppm_wthr) {
1554                 /* clearly missing QoS targets, slow down vrate */
1555                 ioc->busy_level = max(ioc->busy_level, 0);
1556                 ioc->busy_level++;
1557         } else if (rq_wait_pct <= RQ_WAIT_BUSY_PCT * UNBUSY_THR_PCT / 100 &&
1558                    missed_ppm[READ] <= ppm_rthr * UNBUSY_THR_PCT / 100 &&
1559                    missed_ppm[WRITE] <= ppm_wthr * UNBUSY_THR_PCT / 100) {
1560                 /* QoS targets are being met with >25% margin */
1561                 if (nr_shortages) {
1562                         /*
1563                          * We're throttling while the device has spare
1564                          * capacity.  If vrate was being slowed down, stop.
1565                          */
1566                         ioc->busy_level = min(ioc->busy_level, 0);
1567
1568                         /*
1569                          * If there are IOs spanning multiple periods, wait
1570                          * them out before pushing the device harder.  If
1571                          * there are surpluses, let redistribution work it
1572                          * out first.
1573                          */
1574                         if (!nr_lagging && !nr_surpluses)
1575                                 ioc->busy_level--;
1576                 } else {
1577                         /*
1578                          * Nobody is being throttled and the users aren't
1579                          * issuing enough IOs to saturate the device.  We
1580                          * simply don't know how close the device is to
1581                          * saturation.  Coast.
1582                          */
1583                         ioc->busy_level = 0;
1584                 }
1585         } else {
1586                 /* inside the hysterisis margin, we're good */
1587                 ioc->busy_level = 0;
1588         }
1589
1590         ioc->busy_level = clamp(ioc->busy_level, -1000, 1000);
1591
1592         if (ioc->busy_level > 0 || (ioc->busy_level < 0 && !nr_lagging)) {
1593                 u64 vrate = atomic64_read(&ioc->vtime_rate);
1594                 u64 vrate_min = ioc->vrate_min, vrate_max = ioc->vrate_max;
1595
1596                 /* rq_wait signal is always reliable, ignore user vrate_min */
1597                 if (rq_wait_pct > RQ_WAIT_BUSY_PCT)
1598                         vrate_min = VRATE_MIN;
1599
1600                 /*
1601                  * If vrate is out of bounds, apply clamp gradually as the
1602                  * bounds can change abruptly.  Otherwise, apply busy_level
1603                  * based adjustment.
1604                  */
1605                 if (vrate < vrate_min) {
1606                         vrate = div64_u64(vrate * (100 + VRATE_CLAMP_ADJ_PCT),
1607                                           100);
1608                         vrate = min(vrate, vrate_min);
1609                 } else if (vrate > vrate_max) {
1610                         vrate = div64_u64(vrate * (100 - VRATE_CLAMP_ADJ_PCT),
1611                                           100);
1612                         vrate = max(vrate, vrate_max);
1613                 } else {
1614                         int idx = min_t(int, abs(ioc->busy_level),
1615                                         ARRAY_SIZE(vrate_adj_pct) - 1);
1616                         u32 adj_pct = vrate_adj_pct[idx];
1617
1618                         if (ioc->busy_level > 0)
1619                                 adj_pct = 100 - adj_pct;
1620                         else
1621                                 adj_pct = 100 + adj_pct;
1622
1623                         vrate = clamp(DIV64_U64_ROUND_UP(vrate * adj_pct, 100),
1624                                       vrate_min, vrate_max);
1625                 }
1626
1627                 trace_iocost_ioc_vrate_adj(ioc, vrate, missed_ppm, rq_wait_pct,
1628                                            nr_lagging, nr_shortages,
1629                                            nr_surpluses);
1630
1631                 atomic64_set(&ioc->vtime_rate, vrate);
1632                 ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(
1633                         ioc->period_us * vrate * INUSE_MARGIN_PCT, 100);
1634         } else if (ioc->busy_level != prev_busy_level || nr_lagging) {
1635                 trace_iocost_ioc_vrate_adj(ioc, atomic64_read(&ioc->vtime_rate),
1636                                            missed_ppm, rq_wait_pct, nr_lagging,
1637                                            nr_shortages, nr_surpluses);
1638         }
1639
1640         ioc_refresh_params(ioc, false);
1641
1642         /*
1643          * This period is done.  Move onto the next one.  If nothing's
1644          * going on with the device, stop the timer.
1645          */
1646         atomic64_inc(&ioc->cur_period);
1647
1648         if (ioc->running != IOC_STOP) {
1649                 if (!list_empty(&ioc->active_iocgs)) {
1650                         ioc_start_period(ioc, &now);
1651                 } else {
1652                         ioc->busy_level = 0;
1653                         ioc->running = IOC_IDLE;
1654                 }
1655         }
1656
1657         spin_unlock_irq(&ioc->lock);
1658 }
1659
1660 static void calc_vtime_cost_builtin(struct bio *bio, struct ioc_gq *iocg,
1661                                     bool is_merge, u64 *costp)
1662 {
1663         struct ioc *ioc = iocg->ioc;
1664         u64 coef_seqio, coef_randio, coef_page;
1665         u64 pages = max_t(u64, bio_sectors(bio) >> IOC_SECT_TO_PAGE_SHIFT, 1);
1666         u64 seek_pages = 0;
1667         u64 cost = 0;
1668
1669         switch (bio_op(bio)) {
1670         case REQ_OP_READ:
1671                 coef_seqio      = ioc->params.lcoefs[LCOEF_RSEQIO];
1672                 coef_randio     = ioc->params.lcoefs[LCOEF_RRANDIO];
1673                 coef_page       = ioc->params.lcoefs[LCOEF_RPAGE];
1674                 break;
1675         case REQ_OP_WRITE:
1676                 coef_seqio      = ioc->params.lcoefs[LCOEF_WSEQIO];
1677                 coef_randio     = ioc->params.lcoefs[LCOEF_WRANDIO];
1678                 coef_page       = ioc->params.lcoefs[LCOEF_WPAGE];
1679                 break;
1680         default:
1681                 goto out;
1682         }
1683
1684         if (iocg->cursor) {
1685                 seek_pages = abs(bio->bi_iter.bi_sector - iocg->cursor);
1686                 seek_pages >>= IOC_SECT_TO_PAGE_SHIFT;
1687         }
1688
1689         if (!is_merge) {
1690                 if (seek_pages > LCOEF_RANDIO_PAGES) {
1691                         cost += coef_randio;
1692                 } else {
1693                         cost += coef_seqio;
1694                 }
1695         }
1696         cost += pages * coef_page;
1697 out:
1698         *costp = cost;
1699 }
1700
1701 static u64 calc_vtime_cost(struct bio *bio, struct ioc_gq *iocg, bool is_merge)
1702 {
1703         u64 cost;
1704
1705         calc_vtime_cost_builtin(bio, iocg, is_merge, &cost);
1706         return cost;
1707 }
1708
1709 static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio)
1710 {
1711         struct blkcg_gq *blkg = bio->bi_blkg;
1712         struct ioc *ioc = rqos_to_ioc(rqos);
1713         struct ioc_gq *iocg = blkg_to_iocg(blkg);
1714         struct ioc_now now;
1715         struct iocg_wait wait;
1716         u32 hw_active, hw_inuse;
1717         u64 abs_cost, cost, vtime;
1718
1719         /* bypass IOs if disabled or for root cgroup */
1720         if (!ioc->enabled || !iocg->level)
1721                 return;
1722
1723         /* always activate so that even 0 cost IOs get protected to some level */
1724         if (!iocg_activate(iocg, &now))
1725                 return;
1726
1727         /* calculate the absolute vtime cost */
1728         abs_cost = calc_vtime_cost(bio, iocg, false);
1729         if (!abs_cost)
1730                 return;
1731
1732         iocg->cursor = bio_end_sector(bio);
1733
1734         vtime = atomic64_read(&iocg->vtime);
1735         current_hweight(iocg, &hw_active, &hw_inuse);
1736
1737         if (hw_inuse < hw_active &&
1738             time_after_eq64(vtime + ioc->inuse_margin_vtime, now.vnow)) {
1739                 TRACE_IOCG_PATH(inuse_reset, iocg, &now,
1740                                 iocg->inuse, iocg->weight, hw_inuse, hw_active);
1741                 spin_lock_irq(&ioc->lock);
1742                 propagate_active_weight(iocg, iocg->weight, iocg->weight);
1743                 spin_unlock_irq(&ioc->lock);
1744                 current_hweight(iocg, &hw_active, &hw_inuse);
1745         }
1746
1747         cost = abs_cost_to_cost(abs_cost, hw_inuse);
1748
1749         /*
1750          * If no one's waiting and within budget, issue right away.  The
1751          * tests are racy but the races aren't systemic - we only miss once
1752          * in a while which is fine.
1753          */
1754         if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
1755             time_before_eq64(vtime + cost, now.vnow)) {
1756                 iocg_commit_bio(iocg, bio, cost);
1757                 return;
1758         }
1759
1760         /*
1761          * We activated above but w/o any synchronization. Deactivation is
1762          * synchronized with waitq.lock and we won't get deactivated as long
1763          * as we're waiting or has debt, so we're good if we're activated
1764          * here. In the unlikely case that we aren't, just issue the IO.
1765          */
1766         spin_lock_irq(&iocg->waitq.lock);
1767
1768         if (unlikely(list_empty(&iocg->active_list))) {
1769                 spin_unlock_irq(&iocg->waitq.lock);
1770                 iocg_commit_bio(iocg, bio, cost);
1771                 return;
1772         }
1773
1774         /*
1775          * We're over budget. If @bio has to be issued regardless, remember
1776          * the abs_cost instead of advancing vtime. iocg_kick_waitq() will pay
1777          * off the debt before waking more IOs.
1778          *
1779          * This way, the debt is continuously paid off each period with the
1780          * actual budget available to the cgroup. If we just wound vtime, we
1781          * would incorrectly use the current hw_inuse for the entire amount
1782          * which, for example, can lead to the cgroup staying blocked for a
1783          * long time even with substantially raised hw_inuse.
1784          *
1785          * An iocg with vdebt should stay online so that the timer can keep
1786          * deducting its vdebt and [de]activate use_delay mechanism
1787          * accordingly. We don't want to race against the timer trying to
1788          * clear them and leave @iocg inactive w/ dangling use_delay heavily
1789          * penalizing the cgroup and its descendants.
1790          */
1791         if (bio_issue_as_root_blkg(bio) || fatal_signal_pending(current)) {
1792                 iocg->abs_vdebt += abs_cost;
1793                 if (iocg_kick_delay(iocg, &now, cost))
1794                         blkcg_schedule_throttle(rqos->q,
1795                                         (bio->bi_opf & REQ_SWAP) == REQ_SWAP);
1796                 spin_unlock_irq(&iocg->waitq.lock);
1797                 return;
1798         }
1799
1800         /*
1801          * Append self to the waitq and schedule the wakeup timer if we're
1802          * the first waiter.  The timer duration is calculated based on the
1803          * current vrate.  vtime and hweight changes can make it too short
1804          * or too long.  Each wait entry records the absolute cost it's
1805          * waiting for to allow re-evaluation using a custom wait entry.
1806          *
1807          * If too short, the timer simply reschedules itself.  If too long,
1808          * the period timer will notice and trigger wakeups.
1809          *
1810          * All waiters are on iocg->waitq and the wait states are
1811          * synchronized using waitq.lock.
1812          */
1813         init_waitqueue_func_entry(&wait.wait, iocg_wake_fn);
1814         wait.wait.private = current;
1815         wait.bio = bio;
1816         wait.abs_cost = abs_cost;
1817         wait.committed = false; /* will be set true by waker */
1818
1819         __add_wait_queue_entry_tail(&iocg->waitq, &wait.wait);
1820         iocg_kick_waitq(iocg, &now);
1821
1822         spin_unlock_irq(&iocg->waitq.lock);
1823
1824         while (true) {
1825                 set_current_state(TASK_UNINTERRUPTIBLE);
1826                 if (wait.committed)
1827                         break;
1828                 io_schedule();
1829         }
1830
1831         /* waker already committed us, proceed */
1832         finish_wait(&iocg->waitq, &wait.wait);
1833 }
1834
1835 static void ioc_rqos_merge(struct rq_qos *rqos, struct request *rq,
1836                            struct bio *bio)
1837 {
1838         struct ioc_gq *iocg = blkg_to_iocg(bio->bi_blkg);
1839         struct ioc *ioc = iocg->ioc;
1840         sector_t bio_end = bio_end_sector(bio);
1841         struct ioc_now now;
1842         u32 hw_inuse;
1843         u64 abs_cost, cost;
1844         unsigned long flags;
1845
1846         /* bypass if disabled or for root cgroup */
1847         if (!ioc->enabled || !iocg->level)
1848                 return;
1849
1850         abs_cost = calc_vtime_cost(bio, iocg, true);
1851         if (!abs_cost)
1852                 return;
1853
1854         ioc_now(ioc, &now);
1855         current_hweight(iocg, NULL, &hw_inuse);
1856         cost = abs_cost_to_cost(abs_cost, hw_inuse);
1857
1858         /* update cursor if backmerging into the request at the cursor */
1859         if (blk_rq_pos(rq) < bio_end &&
1860             blk_rq_pos(rq) + blk_rq_sectors(rq) == iocg->cursor)
1861                 iocg->cursor = bio_end;
1862
1863         /*
1864          * Charge if there's enough vtime budget and the existing request has
1865          * cost assigned.
1866          */
1867         if (rq->bio && rq->bio->bi_iocost_cost &&
1868             time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {
1869                 iocg_commit_bio(iocg, bio, cost);
1870                 return;
1871         }
1872
1873         /*
1874          * Otherwise, account it as debt if @iocg is online, which it should
1875          * be for the vast majority of cases. See debt handling in
1876          * ioc_rqos_throttle() for details.
1877          */
1878         spin_lock_irqsave(&iocg->waitq.lock, flags);
1879         if (likely(!list_empty(&iocg->active_list))) {
1880                 iocg->abs_vdebt += abs_cost;
1881                 iocg_kick_delay(iocg, &now, cost);
1882         } else {
1883                 iocg_commit_bio(iocg, bio, cost);
1884         }
1885         spin_unlock_irqrestore(&iocg->waitq.lock, flags);
1886 }
1887
1888 static void ioc_rqos_done_bio(struct rq_qos *rqos, struct bio *bio)
1889 {
1890         struct ioc_gq *iocg = blkg_to_iocg(bio->bi_blkg);
1891
1892         if (iocg && bio->bi_iocost_cost)
1893                 atomic64_add(bio->bi_iocost_cost, &iocg->done_vtime);
1894 }
1895
1896 static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
1897 {
1898         struct ioc *ioc = rqos_to_ioc(rqos);
1899         u64 on_q_ns, rq_wait_ns;
1900         int pidx, rw;
1901
1902         if (!ioc->enabled || !rq->alloc_time_ns || !rq->start_time_ns)
1903                 return;
1904
1905         switch (req_op(rq) & REQ_OP_MASK) {
1906         case REQ_OP_READ:
1907                 pidx = QOS_RLAT;
1908                 rw = READ;
1909                 break;
1910         case REQ_OP_WRITE:
1911                 pidx = QOS_WLAT;
1912                 rw = WRITE;
1913                 break;
1914         default:
1915                 return;
1916         }
1917
1918         on_q_ns = ktime_get_ns() - rq->alloc_time_ns;
1919         rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;
1920
1921         if (on_q_ns <= ioc->params.qos[pidx] * NSEC_PER_USEC)
1922                 this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);
1923         else
1924                 this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);
1925
1926         this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);
1927 }
1928
1929 static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)
1930 {
1931         struct ioc *ioc = rqos_to_ioc(rqos);
1932
1933         spin_lock_irq(&ioc->lock);
1934         ioc_refresh_params(ioc, false);
1935         spin_unlock_irq(&ioc->lock);
1936 }
1937
1938 static void ioc_rqos_exit(struct rq_qos *rqos)
1939 {
1940         struct ioc *ioc = rqos_to_ioc(rqos);
1941
1942         blkcg_deactivate_policy(rqos->q, &blkcg_policy_iocost);
1943
1944         spin_lock_irq(&ioc->lock);
1945         ioc->running = IOC_STOP;
1946         spin_unlock_irq(&ioc->lock);
1947
1948         del_timer_sync(&ioc->timer);
1949         free_percpu(ioc->pcpu_stat);
1950         kfree(ioc);
1951 }
1952
1953 static struct rq_qos_ops ioc_rqos_ops = {
1954         .throttle = ioc_rqos_throttle,
1955         .merge = ioc_rqos_merge,
1956         .done_bio = ioc_rqos_done_bio,
1957         .done = ioc_rqos_done,
1958         .queue_depth_changed = ioc_rqos_queue_depth_changed,
1959         .exit = ioc_rqos_exit,
1960 };
1961
1962 static int blk_iocost_init(struct request_queue *q)
1963 {
1964         struct ioc *ioc;
1965         struct rq_qos *rqos;
1966         int ret;
1967
1968         ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
1969         if (!ioc)
1970                 return -ENOMEM;
1971
1972         ioc->pcpu_stat = alloc_percpu(struct ioc_pcpu_stat);
1973         if (!ioc->pcpu_stat) {
1974                 kfree(ioc);
1975                 return -ENOMEM;
1976         }
1977
1978         rqos = &ioc->rqos;
1979         rqos->id = RQ_QOS_COST;
1980         rqos->ops = &ioc_rqos_ops;
1981         rqos->q = q;
1982
1983         spin_lock_init(&ioc->lock);
1984         timer_setup(&ioc->timer, ioc_timer_fn, 0);
1985         INIT_LIST_HEAD(&ioc->active_iocgs);
1986
1987         ioc->running = IOC_IDLE;
1988         atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);
1989         seqcount_init(&ioc->period_seqcount);
1990         ioc->period_at = ktime_to_us(ktime_get());
1991         atomic64_set(&ioc->cur_period, 0);
1992         atomic_set(&ioc->hweight_gen, 0);
1993
1994         spin_lock_irq(&ioc->lock);
1995         ioc->autop_idx = AUTOP_INVALID;
1996         ioc_refresh_params(ioc, true);
1997         spin_unlock_irq(&ioc->lock);
1998
1999         rq_qos_add(q, rqos);
2000         ret = blkcg_activate_policy(q, &blkcg_policy_iocost);
2001         if (ret) {
2002                 rq_qos_del(q, rqos);
2003                 free_percpu(ioc->pcpu_stat);
2004                 kfree(ioc);
2005                 return ret;
2006         }
2007         return 0;
2008 }
2009
2010 static struct blkcg_policy_data *ioc_cpd_alloc(gfp_t gfp)
2011 {
2012         struct ioc_cgrp *iocc;
2013
2014         iocc = kzalloc(sizeof(struct ioc_cgrp), gfp);
2015         if (!iocc)
2016                 return NULL;
2017
2018         iocc->dfl_weight = CGROUP_WEIGHT_DFL;
2019         return &iocc->cpd;
2020 }
2021
2022 static void ioc_cpd_free(struct blkcg_policy_data *cpd)
2023 {
2024         kfree(container_of(cpd, struct ioc_cgrp, cpd));
2025 }
2026
2027 static struct blkg_policy_data *ioc_pd_alloc(gfp_t gfp, struct request_queue *q,
2028                                              struct blkcg *blkcg)
2029 {
2030         int levels = blkcg->css.cgroup->level + 1;
2031         struct ioc_gq *iocg;
2032
2033         iocg = kzalloc_node(sizeof(*iocg) + levels * sizeof(iocg->ancestors[0]),
2034                             gfp, q->node);
2035         if (!iocg)
2036                 return NULL;
2037
2038         return &iocg->pd;
2039 }
2040
2041 static void ioc_pd_init(struct blkg_policy_data *pd)
2042 {
2043         struct ioc_gq *iocg = pd_to_iocg(pd);
2044         struct blkcg_gq *blkg = pd_to_blkg(&iocg->pd);
2045         struct ioc *ioc = q_to_ioc(blkg->q);
2046         struct ioc_now now;
2047         struct blkcg_gq *tblkg;
2048         unsigned long flags;
2049
2050         ioc_now(ioc, &now);
2051
2052         iocg->ioc = ioc;
2053         atomic64_set(&iocg->vtime, now.vnow);
2054         atomic64_set(&iocg->done_vtime, now.vnow);
2055         atomic64_set(&iocg->active_period, atomic64_read(&ioc->cur_period));
2056         INIT_LIST_HEAD(&iocg->active_list);
2057         iocg->hweight_active = HWEIGHT_WHOLE;
2058         iocg->hweight_inuse = HWEIGHT_WHOLE;
2059
2060         init_waitqueue_head(&iocg->waitq);
2061         hrtimer_init(&iocg->waitq_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2062         iocg->waitq_timer.function = iocg_waitq_timer_fn;
2063         hrtimer_init(&iocg->delay_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2064         iocg->delay_timer.function = iocg_delay_timer_fn;
2065
2066         iocg->level = blkg->blkcg->css.cgroup->level;
2067
2068         for (tblkg = blkg; tblkg; tblkg = tblkg->parent) {
2069                 struct ioc_gq *tiocg = blkg_to_iocg(tblkg);
2070                 iocg->ancestors[tiocg->level] = tiocg;
2071         }
2072
2073         spin_lock_irqsave(&ioc->lock, flags);
2074         weight_updated(iocg);
2075         spin_unlock_irqrestore(&ioc->lock, flags);
2076 }
2077
2078 static void ioc_pd_free(struct blkg_policy_data *pd)
2079 {
2080         struct ioc_gq *iocg = pd_to_iocg(pd);
2081         struct ioc *ioc = iocg->ioc;
2082         unsigned long flags;
2083
2084         if (ioc) {
2085                 spin_lock_irqsave(&ioc->lock, flags);
2086                 if (!list_empty(&iocg->active_list)) {
2087                         propagate_active_weight(iocg, 0, 0);
2088                         list_del_init(&iocg->active_list);
2089                 }
2090                 spin_unlock_irqrestore(&ioc->lock, flags);
2091
2092                 hrtimer_cancel(&iocg->waitq_timer);
2093                 hrtimer_cancel(&iocg->delay_timer);
2094         }
2095         kfree(iocg);
2096 }
2097
2098 static u64 ioc_weight_prfill(struct seq_file *sf, struct blkg_policy_data *pd,
2099                              int off)
2100 {
2101         const char *dname = blkg_dev_name(pd->blkg);
2102         struct ioc_gq *iocg = pd_to_iocg(pd);
2103
2104         if (dname && iocg->cfg_weight)
2105                 seq_printf(sf, "%s %u\n", dname, iocg->cfg_weight);
2106         return 0;
2107 }
2108
2109
2110 static int ioc_weight_show(struct seq_file *sf, void *v)
2111 {
2112         struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
2113         struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
2114
2115         seq_printf(sf, "default %u\n", iocc->dfl_weight);
2116         blkcg_print_blkgs(sf, blkcg, ioc_weight_prfill,
2117                           &blkcg_policy_iocost, seq_cft(sf)->private, false);
2118         return 0;
2119 }
2120
2121 static ssize_t ioc_weight_write(struct kernfs_open_file *of, char *buf,
2122                                 size_t nbytes, loff_t off)
2123 {
2124         struct blkcg *blkcg = css_to_blkcg(of_css(of));
2125         struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
2126         struct blkg_conf_ctx ctx;
2127         struct ioc_gq *iocg;
2128         u32 v;
2129         int ret;
2130
2131         if (!strchr(buf, ':')) {
2132                 struct blkcg_gq *blkg;
2133
2134                 if (!sscanf(buf, "default %u", &v) && !sscanf(buf, "%u", &v))
2135                         return -EINVAL;
2136
2137                 if (v < CGROUP_WEIGHT_MIN || v > CGROUP_WEIGHT_MAX)
2138                         return -EINVAL;
2139
2140                 spin_lock(&blkcg->lock);
2141                 iocc->dfl_weight = v;
2142                 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
2143                         struct ioc_gq *iocg = blkg_to_iocg(blkg);
2144
2145                         if (iocg) {
2146                                 spin_lock_irq(&iocg->ioc->lock);
2147                                 weight_updated(iocg);
2148                                 spin_unlock_irq(&iocg->ioc->lock);
2149                         }
2150                 }
2151                 spin_unlock(&blkcg->lock);
2152
2153                 return nbytes;
2154         }
2155
2156         ret = blkg_conf_prep(blkcg, &blkcg_policy_iocost, buf, &ctx);
2157         if (ret)
2158                 return ret;
2159
2160         iocg = blkg_to_iocg(ctx.blkg);
2161
2162         if (!strncmp(ctx.body, "default", 7)) {
2163                 v = 0;
2164         } else {
2165                 if (!sscanf(ctx.body, "%u", &v))
2166                         goto einval;
2167                 if (v < CGROUP_WEIGHT_MIN || v > CGROUP_WEIGHT_MAX)
2168                         goto einval;
2169         }
2170
2171         spin_lock(&iocg->ioc->lock);
2172         iocg->cfg_weight = v;
2173         weight_updated(iocg);
2174         spin_unlock(&iocg->ioc->lock);
2175
2176         blkg_conf_finish(&ctx);
2177         return nbytes;
2178
2179 einval:
2180         blkg_conf_finish(&ctx);
2181         return -EINVAL;
2182 }
2183
2184 static u64 ioc_qos_prfill(struct seq_file *sf, struct blkg_policy_data *pd,
2185                           int off)
2186 {
2187         const char *dname = blkg_dev_name(pd->blkg);
2188         struct ioc *ioc = pd_to_iocg(pd)->ioc;
2189
2190         if (!dname)
2191                 return 0;
2192
2193         seq_printf(sf, "%s enable=%d ctrl=%s rpct=%u.%02u rlat=%u wpct=%u.%02u wlat=%u min=%u.%02u max=%u.%02u\n",
2194                    dname, ioc->enabled, ioc->user_qos_params ? "user" : "auto",
2195                    ioc->params.qos[QOS_RPPM] / 10000,
2196                    ioc->params.qos[QOS_RPPM] % 10000 / 100,
2197                    ioc->params.qos[QOS_RLAT],
2198                    ioc->params.qos[QOS_WPPM] / 10000,
2199                    ioc->params.qos[QOS_WPPM] % 10000 / 100,
2200                    ioc->params.qos[QOS_WLAT],
2201                    ioc->params.qos[QOS_MIN] / 10000,
2202                    ioc->params.qos[QOS_MIN] % 10000 / 100,
2203                    ioc->params.qos[QOS_MAX] / 10000,
2204                    ioc->params.qos[QOS_MAX] % 10000 / 100);
2205         return 0;
2206 }
2207
2208 static int ioc_qos_show(struct seq_file *sf, void *v)
2209 {
2210         struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
2211
2212         blkcg_print_blkgs(sf, blkcg, ioc_qos_prfill,
2213                           &blkcg_policy_iocost, seq_cft(sf)->private, false);
2214         return 0;
2215 }
2216
2217 static const match_table_t qos_ctrl_tokens = {
2218         { QOS_ENABLE,           "enable=%u"     },
2219         { QOS_CTRL,             "ctrl=%s"       },
2220         { NR_QOS_CTRL_PARAMS,   NULL            },
2221 };
2222
2223 static const match_table_t qos_tokens = {
2224         { QOS_RPPM,             "rpct=%s"       },
2225         { QOS_RLAT,             "rlat=%u"       },
2226         { QOS_WPPM,             "wpct=%s"       },
2227         { QOS_WLAT,             "wlat=%u"       },
2228         { QOS_MIN,              "min=%s"        },
2229         { QOS_MAX,              "max=%s"        },
2230         { NR_QOS_PARAMS,        NULL            },
2231 };
2232
2233 static ssize_t ioc_qos_write(struct kernfs_open_file *of, char *input,
2234                              size_t nbytes, loff_t off)
2235 {
2236         struct gendisk *disk;
2237         struct ioc *ioc;
2238         u32 qos[NR_QOS_PARAMS];
2239         bool enable, user;
2240         char *p;
2241         int ret;
2242
2243         disk = blkcg_conf_get_disk(&input);
2244         if (IS_ERR(disk))
2245                 return PTR_ERR(disk);
2246
2247         ioc = q_to_ioc(disk->queue);
2248         if (!ioc) {
2249                 ret = blk_iocost_init(disk->queue);
2250                 if (ret)
2251                         goto err;
2252                 ioc = q_to_ioc(disk->queue);
2253         }
2254
2255         spin_lock_irq(&ioc->lock);
2256         memcpy(qos, ioc->params.qos, sizeof(qos));
2257         enable = ioc->enabled;
2258         user = ioc->user_qos_params;
2259         spin_unlock_irq(&ioc->lock);
2260
2261         while ((p = strsep(&input, " \t\n"))) {
2262                 substring_t args[MAX_OPT_ARGS];
2263                 char buf[32];
2264                 int tok;
2265                 s64 v;
2266
2267                 if (!*p)
2268                         continue;
2269
2270                 switch (match_token(p, qos_ctrl_tokens, args)) {
2271                 case QOS_ENABLE:
2272                         match_u64(&args[0], &v);
2273                         enable = v;
2274                         continue;
2275                 case QOS_CTRL:
2276                         match_strlcpy(buf, &args[0], sizeof(buf));
2277                         if (!strcmp(buf, "auto"))
2278                                 user = false;
2279                         else if (!strcmp(buf, "user"))
2280                                 user = true;
2281                         else
2282                                 goto einval;
2283                         continue;
2284                 }
2285
2286                 tok = match_token(p, qos_tokens, args);
2287                 switch (tok) {
2288                 case QOS_RPPM:
2289                 case QOS_WPPM:
2290                         if (match_strlcpy(buf, &args[0], sizeof(buf)) >=
2291                             sizeof(buf))
2292                                 goto einval;
2293                         if (cgroup_parse_float(buf, 2, &v))
2294                                 goto einval;
2295                         if (v < 0 || v > 10000)
2296                                 goto einval;
2297                         qos[tok] = v * 100;
2298                         break;
2299                 case QOS_RLAT:
2300                 case QOS_WLAT:
2301                         if (match_u64(&args[0], &v))
2302                                 goto einval;
2303                         qos[tok] = v;
2304                         break;
2305                 case QOS_MIN:
2306                 case QOS_MAX:
2307                         if (match_strlcpy(buf, &args[0], sizeof(buf)) >=
2308                             sizeof(buf))
2309                                 goto einval;
2310                         if (cgroup_parse_float(buf, 2, &v))
2311                                 goto einval;
2312                         if (v < 0)
2313                                 goto einval;
2314                         qos[tok] = clamp_t(s64, v * 100,
2315                                            VRATE_MIN_PPM, VRATE_MAX_PPM);
2316                         break;
2317                 default:
2318                         goto einval;
2319                 }
2320                 user = true;
2321         }
2322
2323         if (qos[QOS_MIN] > qos[QOS_MAX])
2324                 goto einval;
2325
2326         spin_lock_irq(&ioc->lock);
2327
2328         if (enable) {
2329                 blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
2330                 ioc->enabled = true;
2331         } else {
2332                 blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
2333                 ioc->enabled = false;
2334         }
2335
2336         if (user) {
2337                 memcpy(ioc->params.qos, qos, sizeof(qos));
2338                 ioc->user_qos_params = true;
2339         } else {
2340                 ioc->user_qos_params = false;
2341         }
2342
2343         ioc_refresh_params(ioc, true);
2344         spin_unlock_irq(&ioc->lock);
2345
2346         put_disk_and_module(disk);
2347         return nbytes;
2348 einval:
2349         ret = -EINVAL;
2350 err:
2351         put_disk_and_module(disk);
2352         return ret;
2353 }
2354
2355 static u64 ioc_cost_model_prfill(struct seq_file *sf,
2356                                  struct blkg_policy_data *pd, int off)
2357 {
2358         const char *dname = blkg_dev_name(pd->blkg);
2359         struct ioc *ioc = pd_to_iocg(pd)->ioc;
2360         u64 *u = ioc->params.i_lcoefs;
2361
2362         if (!dname)
2363                 return 0;
2364
2365         seq_printf(sf, "%s ctrl=%s model=linear "
2366                    "rbps=%llu rseqiops=%llu rrandiops=%llu "
2367                    "wbps=%llu wseqiops=%llu wrandiops=%llu\n",
2368                    dname, ioc->user_cost_model ? "user" : "auto",
2369                    u[I_LCOEF_RBPS], u[I_LCOEF_RSEQIOPS], u[I_LCOEF_RRANDIOPS],
2370                    u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS]);
2371         return 0;
2372 }
2373
2374 static int ioc_cost_model_show(struct seq_file *sf, void *v)
2375 {
2376         struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
2377
2378         blkcg_print_blkgs(sf, blkcg, ioc_cost_model_prfill,
2379                           &blkcg_policy_iocost, seq_cft(sf)->private, false);
2380         return 0;
2381 }
2382
2383 static const match_table_t cost_ctrl_tokens = {
2384         { COST_CTRL,            "ctrl=%s"       },
2385         { COST_MODEL,           "model=%s"      },
2386         { NR_COST_CTRL_PARAMS,  NULL            },
2387 };
2388
2389 static const match_table_t i_lcoef_tokens = {
2390         { I_LCOEF_RBPS,         "rbps=%u"       },
2391         { I_LCOEF_RSEQIOPS,     "rseqiops=%u"   },
2392         { I_LCOEF_RRANDIOPS,    "rrandiops=%u"  },
2393         { I_LCOEF_WBPS,         "wbps=%u"       },
2394         { I_LCOEF_WSEQIOPS,     "wseqiops=%u"   },
2395         { I_LCOEF_WRANDIOPS,    "wrandiops=%u"  },
2396         { NR_I_LCOEFS,          NULL            },
2397 };
2398
2399 static ssize_t ioc_cost_model_write(struct kernfs_open_file *of, char *input,
2400                                     size_t nbytes, loff_t off)
2401 {
2402         struct gendisk *disk;
2403         struct ioc *ioc;
2404         u64 u[NR_I_LCOEFS];
2405         bool user;
2406         char *p;
2407         int ret;
2408
2409         disk = blkcg_conf_get_disk(&input);
2410         if (IS_ERR(disk))
2411                 return PTR_ERR(disk);
2412
2413         ioc = q_to_ioc(disk->queue);
2414         if (!ioc) {
2415                 ret = blk_iocost_init(disk->queue);
2416                 if (ret)
2417                         goto err;
2418                 ioc = q_to_ioc(disk->queue);
2419         }
2420
2421         spin_lock_irq(&ioc->lock);
2422         memcpy(u, ioc->params.i_lcoefs, sizeof(u));
2423         user = ioc->user_cost_model;
2424         spin_unlock_irq(&ioc->lock);
2425
2426         while ((p = strsep(&input, " \t\n"))) {
2427                 substring_t args[MAX_OPT_ARGS];
2428                 char buf[32];
2429                 int tok;
2430                 u64 v;
2431
2432                 if (!*p)
2433                         continue;
2434
2435                 switch (match_token(p, cost_ctrl_tokens, args)) {
2436                 case COST_CTRL:
2437                         match_strlcpy(buf, &args[0], sizeof(buf));
2438                         if (!strcmp(buf, "auto"))
2439                                 user = false;
2440                         else if (!strcmp(buf, "user"))
2441                                 user = true;
2442                         else
2443                                 goto einval;
2444                         continue;
2445                 case COST_MODEL:
2446                         match_strlcpy(buf, &args[0], sizeof(buf));
2447                         if (strcmp(buf, "linear"))
2448                                 goto einval;
2449                         continue;
2450                 }
2451
2452                 tok = match_token(p, i_lcoef_tokens, args);
2453                 if (tok == NR_I_LCOEFS)
2454                         goto einval;
2455                 if (match_u64(&args[0], &v))
2456                         goto einval;
2457                 u[tok] = v;
2458                 user = true;
2459         }
2460
2461         spin_lock_irq(&ioc->lock);
2462         if (user) {
2463                 memcpy(ioc->params.i_lcoefs, u, sizeof(u));
2464                 ioc->user_cost_model = true;
2465         } else {
2466                 ioc->user_cost_model = false;
2467         }
2468         ioc_refresh_params(ioc, true);
2469         spin_unlock_irq(&ioc->lock);
2470
2471         put_disk_and_module(disk);
2472         return nbytes;
2473
2474 einval:
2475         ret = -EINVAL;
2476 err:
2477         put_disk_and_module(disk);
2478         return ret;
2479 }
2480
2481 static struct cftype ioc_files[] = {
2482         {
2483                 .name = "weight",
2484                 .flags = CFTYPE_NOT_ON_ROOT,
2485                 .seq_show = ioc_weight_show,
2486                 .write = ioc_weight_write,
2487         },
2488         {
2489                 .name = "cost.qos",
2490                 .flags = CFTYPE_ONLY_ON_ROOT,
2491                 .seq_show = ioc_qos_show,
2492                 .write = ioc_qos_write,
2493         },
2494         {
2495                 .name = "cost.model",
2496                 .flags = CFTYPE_ONLY_ON_ROOT,
2497                 .seq_show = ioc_cost_model_show,
2498                 .write = ioc_cost_model_write,
2499         },
2500         {}
2501 };
2502
2503 static struct blkcg_policy blkcg_policy_iocost = {
2504         .dfl_cftypes    = ioc_files,
2505         .cpd_alloc_fn   = ioc_cpd_alloc,
2506         .cpd_free_fn    = ioc_cpd_free,
2507         .pd_alloc_fn    = ioc_pd_alloc,
2508         .pd_init_fn     = ioc_pd_init,
2509         .pd_free_fn     = ioc_pd_free,
2510 };
2511
2512 static int __init ioc_init(void)
2513 {
2514         return blkcg_policy_register(&blkcg_policy_iocost);
2515 }
2516
2517 static void __exit ioc_exit(void)
2518 {
2519         return blkcg_policy_unregister(&blkcg_policy_iocost);
2520 }
2521
2522 module_init(ioc_init);
2523 module_exit(ioc_exit);