drivers/pmdomain/qcom/cpr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
   4  * Copyright (c) 2019, Linaro Limited
   5  */
   6
   7 #include <linux/module.h>
   8 #include <linux/err.h>
   9 #include <linux/debugfs.h>
  10 #include <linux/string.h>
  11 #include <linux/kernel.h>
  12 #include <linux/list.h>
  13 #include <linux/init.h>
  14 #include <linux/io.h>
  15 #include <linux/bitops.h>
  16 #include <linux/slab.h>
  17 #include <linux/of.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/pm_domain.h>
  20 #include <linux/pm_opp.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/regmap.h>
  23 #include <linux/mfd/syscon.h>
  24 #include <linux/regulator/consumer.h>
  25 #include <linux/clk.h>
  26 #include <linux/nvmem-consumer.h>
  27
  28 /* Register Offsets for RB-CPR and Bit Definitions */
  29
  30 /* RBCPR Version Register */
  31 #define REG_RBCPR_VERSION               0
  32 #define RBCPR_VER_2                     0x02
  33 #define FLAGS_IGNORE_1ST_IRQ_STATUS     BIT(0)
  34
  35 /* RBCPR Gate Count and Target Registers */
  36 #define REG_RBCPR_GCNT_TARGET(n)        (0x60 + 4 * (n))
  37
  38 #define RBCPR_GCNT_TARGET_TARGET_SHIFT  0
  39 #define RBCPR_GCNT_TARGET_TARGET_MASK   GENMASK(11, 0)
  40 #define RBCPR_GCNT_TARGET_GCNT_SHIFT    12
  41 #define RBCPR_GCNT_TARGET_GCNT_MASK     GENMASK(9, 0)
  42
  43 /* RBCPR Timer Control */
  44 #define REG_RBCPR_TIMER_INTERVAL        0x44
  45 #define REG_RBIF_TIMER_ADJUST           0x4c
  46
  47 #define RBIF_TIMER_ADJ_CONS_UP_MASK     GENMASK(3, 0)
  48 #define RBIF_TIMER_ADJ_CONS_UP_SHIFT    0
  49 #define RBIF_TIMER_ADJ_CONS_DOWN_MASK   GENMASK(3, 0)
  50 #define RBIF_TIMER_ADJ_CONS_DOWN_SHIFT  4
  51 #define RBIF_TIMER_ADJ_CLAMP_INT_MASK   GENMASK(7, 0)
  52 #define RBIF_TIMER_ADJ_CLAMP_INT_SHIFT  8
  53
  54 /* RBCPR Config Register */
  55 #define REG_RBIF_LIMIT                  0x48
  56 #define RBIF_LIMIT_CEILING_MASK         GENMASK(5, 0)
  57 #define RBIF_LIMIT_CEILING_SHIFT        6
  58 #define RBIF_LIMIT_FLOOR_BITS           6
  59 #define RBIF_LIMIT_FLOOR_MASK           GENMASK(5, 0)
  60
  61 #define RBIF_LIMIT_CEILING_DEFAULT      RBIF_LIMIT_CEILING_MASK
  62 #define RBIF_LIMIT_FLOOR_DEFAULT        0
  63
  64 #define REG_RBIF_SW_VLEVEL              0x94
  65 #define RBIF_SW_VLEVEL_DEFAULT          0x20
  66
  67 #define REG_RBCPR_STEP_QUOT             0x80
  68 #define RBCPR_STEP_QUOT_STEPQUOT_MASK   GENMASK(7, 0)
  69 #define RBCPR_STEP_QUOT_IDLE_CLK_MASK   GENMASK(3, 0)
  70 #define RBCPR_STEP_QUOT_IDLE_CLK_SHIFT  8
  71
  72 /* RBCPR Control Register */
  73 #define REG_RBCPR_CTL                   0x90
  74
  75 #define RBCPR_CTL_LOOP_EN                       BIT(0)
  76 #define RBCPR_CTL_TIMER_EN                      BIT(3)
  77 #define RBCPR_CTL_SW_AUTO_CONT_ACK_EN           BIT(5)
  78 #define RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN       BIT(6)
  79 #define RBCPR_CTL_COUNT_MODE                    BIT(10)
  80 #define RBCPR_CTL_UP_THRESHOLD_MASK     GENMASK(3, 0)
  81 #define RBCPR_CTL_UP_THRESHOLD_SHIFT    24
  82 #define RBCPR_CTL_DN_THRESHOLD_MASK     GENMASK(3, 0)
  83 #define RBCPR_CTL_DN_THRESHOLD_SHIFT    28
  84
  85 /* RBCPR Ack/Nack Response */
  86 #define REG_RBIF_CONT_ACK_CMD           0x98
  87 #define REG_RBIF_CONT_NACK_CMD          0x9c
  88
  89 /* RBCPR Result status Register */
  90 #define REG_RBCPR_RESULT_0              0xa0
  91
  92 #define RBCPR_RESULT0_BUSY_SHIFT        19
  93 #define RBCPR_RESULT0_BUSY_MASK         BIT(RBCPR_RESULT0_BUSY_SHIFT)
  94 #define RBCPR_RESULT0_ERROR_LT0_SHIFT   18
  95 #define RBCPR_RESULT0_ERROR_SHIFT       6
  96 #define RBCPR_RESULT0_ERROR_MASK        GENMASK(11, 0)
  97 #define RBCPR_RESULT0_ERROR_STEPS_SHIFT 2
  98 #define RBCPR_RESULT0_ERROR_STEPS_MASK  GENMASK(3, 0)
  99 #define RBCPR_RESULT0_STEP_UP_SHIFT     1
 100
 101 /* RBCPR Interrupt Control Register */
 102 #define REG_RBIF_IRQ_EN(n)              (0x100 + 4 * (n))
 103 #define REG_RBIF_IRQ_CLEAR              0x110
 104 #define REG_RBIF_IRQ_STATUS             0x114
 105
 106 #define CPR_INT_DONE            BIT(0)
 107 #define CPR_INT_MIN             BIT(1)
 108 #define CPR_INT_DOWN            BIT(2)
 109 #define CPR_INT_MID             BIT(3)
 110 #define CPR_INT_UP              BIT(4)
 111 #define CPR_INT_MAX             BIT(5)
 112 #define CPR_INT_CLAMP           BIT(6)
 113 #define CPR_INT_ALL     (CPR_INT_DONE | CPR_INT_MIN | CPR_INT_DOWN | \
 114                         CPR_INT_MID | CPR_INT_UP | CPR_INT_MAX | CPR_INT_CLAMP)
 115 #define CPR_INT_DEFAULT (CPR_INT_UP | CPR_INT_DOWN)
 116
 117 #define CPR_NUM_RING_OSC        8
 118
 119 /* CPR eFuse parameters */
 120 #define CPR_FUSE_TARGET_QUOT_BITS_MASK  GENMASK(11, 0)
 121
 122 #define CPR_FUSE_MIN_QUOT_DIFF          50
 123
 124 #define FUSE_REVISION_UNKNOWN           (-1)
 125
 126 enum voltage_change_dir {
 127         NO_CHANGE,
 128         DOWN,
 129         UP,
 130 };
 131
 132 struct cpr_fuse {
 133         char *ring_osc;
 134         char *init_voltage;
 135         char *quotient;
 136         char *quotient_offset;
 137 };
 138
 139 struct fuse_corner_data {
 140         int ref_uV;
 141         int max_uV;
 142         int min_uV;
 143         int max_volt_scale;
 144         int max_quot_scale;
 145         /* fuse quot */
 146         int quot_offset;
 147         int quot_scale;
 148         int quot_adjust;
 149         /* fuse quot_offset */
 150         int quot_offset_scale;
 151         int quot_offset_adjust;
 152 };
 153
 154 struct cpr_fuses {
 155         int init_voltage_step;
 156         int init_voltage_width;
 157         struct fuse_corner_data *fuse_corner_data;
 158 };
 159
 160 struct corner_data {
 161         unsigned int fuse_corner;
 162         unsigned long freq;
 163 };
 164
 165 struct cpr_desc {
 166         unsigned int num_fuse_corners;
 167         int min_diff_quot;
 168         int *step_quot;
 169
 170         unsigned int            timer_delay_us;
 171         unsigned int            timer_cons_up;
 172         unsigned int            timer_cons_down;
 173         unsigned int            up_threshold;
 174         unsigned int            down_threshold;
 175         unsigned int            idle_clocks;
 176         unsigned int            gcnt_us;
 177         unsigned int            vdd_apc_step_up_limit;
 178         unsigned int            vdd_apc_step_down_limit;
 179         unsigned int            clamp_timer_interval;
 180
 181         struct cpr_fuses cpr_fuses;
 182         bool reduce_to_fuse_uV;
 183         bool reduce_to_corner_uV;
 184 };
 185
 186 struct acc_desc {
 187         unsigned int    enable_reg;
 188         u32             enable_mask;
 189
 190         struct reg_sequence     *config;
 191         struct reg_sequence     *settings;
 192         int                     num_regs_per_fuse;
 193 };
 194
 195 struct cpr_acc_desc {
 196         const struct cpr_desc *cpr_desc;
 197         const struct acc_desc *acc_desc;
 198 };
 199
 200 struct fuse_corner {
 201         int min_uV;
 202         int max_uV;
 203         int uV;
 204         int quot;
 205         int step_quot;
 206         const struct reg_sequence *accs;
 207         int num_accs;
 208         unsigned long max_freq;
 209         u8 ring_osc_idx;
 210 };
 211
 212 struct corner {
 213         int min_uV;
 214         int max_uV;
 215         int uV;
 216         int last_uV;
 217         int quot_adjust;
 218         u32 save_ctl;
 219         u32 save_irq;
 220         unsigned long freq;
 221         struct fuse_corner *fuse_corner;
 222 };
 223
 224 struct cpr_drv {
 225         unsigned int            num_corners;
 226         unsigned int            ref_clk_khz;
 227
 228         struct generic_pm_domain pd;
 229         struct device           *dev;
 230         struct device           *attached_cpu_dev;
 231         struct mutex            lock;
 232         void __iomem            *base;
 233         struct corner           *corner;
 234         struct regulator        *vdd_apc;
 235         struct clk              *cpu_clk;
 236         struct regmap           *tcsr;
 237         bool                    loop_disabled;
 238         u32                     gcnt;
 239         unsigned long           flags;
 240
 241         struct fuse_corner      *fuse_corners;
 242         struct corner           *corners;
 243
 244         const struct cpr_desc *desc;
 245         const struct acc_desc *acc_desc;
 246         const struct cpr_fuse *cpr_fuses;
 247
 248         struct dentry *debugfs;
 249 };
 250
 251 static bool cpr_is_allowed(struct cpr_drv *drv)
 252 {
 253         return !drv->loop_disabled;
 254 }
 255
 256 static void cpr_write(struct cpr_drv *drv, u32 offset, u32 value)
 257 {
 258         writel_relaxed(value, drv->base + offset);
 259 }
 260
 261 static u32 cpr_read(struct cpr_drv *drv, u32 offset)
 262 {
 263         return readl_relaxed(drv->base + offset);
 264 }
 265
 266 static void
 267 cpr_masked_write(struct cpr_drv *drv, u32 offset, u32 mask, u32 value)
 268 {
 269         u32 val;
 270
 271         val = readl_relaxed(drv->base + offset);
 272         val &= ~mask;
 273         val |= value & mask;
 274         writel_relaxed(val, drv->base + offset);
 275 }
 276
 277 static void cpr_irq_clr(struct cpr_drv *drv)
 278 {
 279         cpr_write(drv, REG_RBIF_IRQ_CLEAR, CPR_INT_ALL);
 280 }
 281
 282 static void cpr_irq_clr_nack(struct cpr_drv *drv)
 283 {
 284         cpr_irq_clr(drv);
 285         cpr_write(drv, REG_RBIF_CONT_NACK_CMD, 1);
 286 }
 287
 288 static void cpr_irq_clr_ack(struct cpr_drv *drv)
 289 {
 290         cpr_irq_clr(drv);
 291         cpr_write(drv, REG_RBIF_CONT_ACK_CMD, 1);
 292 }
 293
 294 static void cpr_irq_set(struct cpr_drv *drv, u32 int_bits)
 295 {
 296         cpr_write(drv, REG_RBIF_IRQ_EN(0), int_bits);
 297 }
 298
 299 static void cpr_ctl_modify(struct cpr_drv *drv, u32 mask, u32 value)
 300 {
 301         cpr_masked_write(drv, REG_RBCPR_CTL, mask, value);
 302 }
 303
 304 static void cpr_ctl_enable(struct cpr_drv *drv, struct corner *corner)
 305 {
 306         u32 val, mask;
 307         const struct cpr_desc *desc = drv->desc;
 308
 309         /* Program Consecutive Up & Down */
 310         val = desc->timer_cons_down << RBIF_TIMER_ADJ_CONS_DOWN_SHIFT;
 311         val |= desc->timer_cons_up << RBIF_TIMER_ADJ_CONS_UP_SHIFT;
 312         mask = RBIF_TIMER_ADJ_CONS_UP_MASK | RBIF_TIMER_ADJ_CONS_DOWN_MASK;
 313         cpr_masked_write(drv, REG_RBIF_TIMER_ADJUST, mask, val);
 314         cpr_masked_write(drv, REG_RBCPR_CTL,
 315                          RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN |
 316                          RBCPR_CTL_SW_AUTO_CONT_ACK_EN,
 317                          corner->save_ctl);
 318         cpr_irq_set(drv, corner->save_irq);
 319
 320         if (cpr_is_allowed(drv) && corner->max_uV > corner->min_uV)
 321                 val = RBCPR_CTL_LOOP_EN;
 322         else
 323                 val = 0;
 324         cpr_ctl_modify(drv, RBCPR_CTL_LOOP_EN, val);
 325 }
 326
 327 static void cpr_ctl_disable(struct cpr_drv *drv)
 328 {
 329         cpr_irq_set(drv, 0);
 330         cpr_ctl_modify(drv, RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN |
 331                        RBCPR_CTL_SW_AUTO_CONT_ACK_EN, 0);
 332         cpr_masked_write(drv, REG_RBIF_TIMER_ADJUST,
 333                          RBIF_TIMER_ADJ_CONS_UP_MASK |
 334                          RBIF_TIMER_ADJ_CONS_DOWN_MASK, 0);
 335         cpr_irq_clr(drv);
 336         cpr_write(drv, REG_RBIF_CONT_ACK_CMD, 1);
 337         cpr_write(drv, REG_RBIF_CONT_NACK_CMD, 1);
 338         cpr_ctl_modify(drv, RBCPR_CTL_LOOP_EN, 0);
 339 }
 340
 341 static bool cpr_ctl_is_enabled(struct cpr_drv *drv)
 342 {
 343         u32 reg_val;
 344
 345         reg_val = cpr_read(drv, REG_RBCPR_CTL);
 346         return reg_val & RBCPR_CTL_LOOP_EN;
 347 }
 348
 349 static bool cpr_ctl_is_busy(struct cpr_drv *drv)
 350 {
 351         u32 reg_val;
 352
 353         reg_val = cpr_read(drv, REG_RBCPR_RESULT_0);
 354         return reg_val & RBCPR_RESULT0_BUSY_MASK;
 355 }
 356
 357 static void cpr_corner_save(struct cpr_drv *drv, struct corner *corner)
 358 {
 359         corner->save_ctl = cpr_read(drv, REG_RBCPR_CTL);
 360         corner->save_irq = cpr_read(drv, REG_RBIF_IRQ_EN(0));
 361 }
 362
 363 static void cpr_corner_restore(struct cpr_drv *drv, struct corner *corner)
 364 {
 365         u32 gcnt, ctl, irq, ro_sel, step_quot;
 366         struct fuse_corner *fuse = corner->fuse_corner;
 367         const struct cpr_desc *desc = drv->desc;
 368         int i;
 369
 370         ro_sel = fuse->ring_osc_idx;
 371         gcnt = drv->gcnt;
 372         gcnt |= fuse->quot - corner->quot_adjust;
 373
 374         /* Program the step quotient and idle clocks */
 375         step_quot = desc->idle_clocks << RBCPR_STEP_QUOT_IDLE_CLK_SHIFT;
 376         step_quot |= fuse->step_quot & RBCPR_STEP_QUOT_STEPQUOT_MASK;
 377         cpr_write(drv, REG_RBCPR_STEP_QUOT, step_quot);
 378
 379         /* Clear the target quotient value and gate count of all ROs */
 380         for (i = 0; i < CPR_NUM_RING_OSC; i++)
 381                 cpr_write(drv, REG_RBCPR_GCNT_TARGET(i), 0);
 382
 383         cpr_write(drv, REG_RBCPR_GCNT_TARGET(ro_sel), gcnt);
 384         ctl = corner->save_ctl;
 385         cpr_write(drv, REG_RBCPR_CTL, ctl);
 386         irq = corner->save_irq;
 387         cpr_irq_set(drv, irq);
 388         dev_dbg(drv->dev, "gcnt = %#08x, ctl = %#08x, irq = %#08x\n", gcnt,
 389                 ctl, irq);
 390 }
 391
 392 static void cpr_set_acc(struct regmap *tcsr, struct fuse_corner *f,
 393                         struct fuse_corner *end)
 394 {
 395         if (f == end)
 396                 return;
 397
 398         if (f < end) {
 399                 for (f += 1; f <= end; f++)
 400                         regmap_multi_reg_write(tcsr, f->accs, f->num_accs);
 401         } else {
 402                 for (f -= 1; f >= end; f--)
 403                         regmap_multi_reg_write(tcsr, f->accs, f->num_accs);
 404         }
 405 }
 406
 407 static int cpr_pre_voltage(struct cpr_drv *drv,
 408                            struct fuse_corner *fuse_corner,
 409                            enum voltage_change_dir dir)
 410 {
 411         struct fuse_corner *prev_fuse_corner = drv->corner->fuse_corner;
 412
 413         if (drv->tcsr && dir == DOWN)
 414                 cpr_set_acc(drv->tcsr, prev_fuse_corner, fuse_corner);
 415
 416         return 0;
 417 }
 418
 419 static int cpr_post_voltage(struct cpr_drv *drv,
 420                             struct fuse_corner *fuse_corner,
 421                             enum voltage_change_dir dir)
 422 {
 423         struct fuse_corner *prev_fuse_corner = drv->corner->fuse_corner;
 424
 425         if (drv->tcsr && dir == UP)
 426                 cpr_set_acc(drv->tcsr, prev_fuse_corner, fuse_corner);
 427
 428         return 0;
 429 }
 430
 431 static int cpr_scale_voltage(struct cpr_drv *drv, struct corner *corner,
 432                              int new_uV, enum voltage_change_dir dir)
 433 {
 434         int ret;
 435         struct fuse_corner *fuse_corner = corner->fuse_corner;
 436
 437         ret = cpr_pre_voltage(drv, fuse_corner, dir);
 438         if (ret)
 439                 return ret;
 440
 441         ret = regulator_set_voltage(drv->vdd_apc, new_uV, new_uV);
 442         if (ret) {
 443                 dev_err_ratelimited(drv->dev, "failed to set apc voltage %d\n",
 444                                     new_uV);
 445                 return ret;
 446         }
 447
 448         ret = cpr_post_voltage(drv, fuse_corner, dir);
 449         if (ret)
 450                 return ret;
 451
 452         return 0;
 453 }
 454
 455 static unsigned int cpr_get_cur_perf_state(struct cpr_drv *drv)
 456 {
 457         return drv->corner ? drv->corner - drv->corners + 1 : 0;
 458 }
 459
 460 static int cpr_scale(struct cpr_drv *drv, enum voltage_change_dir dir)
 461 {
 462         u32 val, error_steps, reg_mask;
 463         int last_uV, new_uV, step_uV, ret;
 464         struct corner *corner;
 465         const struct cpr_desc *desc = drv->desc;
 466
 467         if (dir != UP && dir != DOWN)
 468                 return 0;
 469
 470         step_uV = regulator_get_linear_step(drv->vdd_apc);
 471         if (!step_uV)
 472                 return -EINVAL;
 473
 474         corner = drv->corner;
 475
 476         val = cpr_read(drv, REG_RBCPR_RESULT_0);
 477
 478         error_steps = val >> RBCPR_RESULT0_ERROR_STEPS_SHIFT;
 479         error_steps &= RBCPR_RESULT0_ERROR_STEPS_MASK;
 480         last_uV = corner->last_uV;
 481
 482         if (dir == UP) {
 483                 if (desc->clamp_timer_interval &&
 484                     error_steps < desc->up_threshold) {
 485                         /*
 486                          * Handle the case where another measurement started
 487                          * after the interrupt was triggered due to a core
 488                          * exiting from power collapse.
 489                          */
 490                         error_steps = max(desc->up_threshold,
 491                                           desc->vdd_apc_step_up_limit);
 492                 }
 493
 494                 if (last_uV >= corner->max_uV) {
 495                         cpr_irq_clr_nack(drv);
 496
 497                         /* Maximize the UP threshold */
 498                         reg_mask = RBCPR_CTL_UP_THRESHOLD_MASK;
 499                         reg_mask <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 500                         val = reg_mask;
 501                         cpr_ctl_modify(drv, reg_mask, val);
 502
 503                         /* Disable UP interrupt */
 504                         cpr_irq_set(drv, CPR_INT_DEFAULT & ~CPR_INT_UP);
 505
 506                         return 0;
 507                 }
 508
 509                 if (error_steps > desc->vdd_apc_step_up_limit)
 510                         error_steps = desc->vdd_apc_step_up_limit;
 511
 512                 /* Calculate new voltage */
 513                 new_uV = last_uV + error_steps * step_uV;
 514                 new_uV = min(new_uV, corner->max_uV);
 515
 516                 dev_dbg(drv->dev,
 517                         "UP: -> new_uV: %d last_uV: %d perf state: %u\n",
 518                         new_uV, last_uV, cpr_get_cur_perf_state(drv));
 519         } else {
 520                 if (desc->clamp_timer_interval &&
 521                     error_steps < desc->down_threshold) {
 522                         /*
 523                          * Handle the case where another measurement started
 524                          * after the interrupt was triggered due to a core
 525                          * exiting from power collapse.
 526                          */
 527                         error_steps = max(desc->down_threshold,
 528                                           desc->vdd_apc_step_down_limit);
 529                 }
 530
 531                 if (last_uV <= corner->min_uV) {
 532                         cpr_irq_clr_nack(drv);
 533
 534                         /* Enable auto nack down */
 535                         reg_mask = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 536                         val = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 537
 538                         cpr_ctl_modify(drv, reg_mask, val);
 539
 540                         /* Disable DOWN interrupt */
 541                         cpr_irq_set(drv, CPR_INT_DEFAULT & ~CPR_INT_DOWN);
 542
 543                         return 0;
 544                 }
 545
 546                 if (error_steps > desc->vdd_apc_step_down_limit)
 547                         error_steps = desc->vdd_apc_step_down_limit;
 548
 549                 /* Calculate new voltage */
 550                 new_uV = last_uV - error_steps * step_uV;
 551                 new_uV = max(new_uV, corner->min_uV);
 552
 553                 dev_dbg(drv->dev,
 554                         "DOWN: -> new_uV: %d last_uV: %d perf state: %u\n",
 555                         new_uV, last_uV, cpr_get_cur_perf_state(drv));
 556         }
 557
 558         ret = cpr_scale_voltage(drv, corner, new_uV, dir);
 559         if (ret) {
 560                 cpr_irq_clr_nack(drv);
 561                 return ret;
 562         }
 563         drv->corner->last_uV = new_uV;
 564
 565         if (dir == UP) {
 566                 /* Disable auto nack down */
 567                 reg_mask = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 568                 val = 0;
 569         } else {
 570                 /* Restore default threshold for UP */
 571                 reg_mask = RBCPR_CTL_UP_THRESHOLD_MASK;
 572                 reg_mask <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 573                 val = desc->up_threshold;
 574                 val <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 575         }
 576
 577         cpr_ctl_modify(drv, reg_mask, val);
 578
 579         /* Re-enable default interrupts */
 580         cpr_irq_set(drv, CPR_INT_DEFAULT);
 581
 582         /* Ack */
 583         cpr_irq_clr_ack(drv);
 584
 585         return 0;
 586 }
 587
 588 static irqreturn_t cpr_irq_handler(int irq, void *dev)
 589 {
 590         struct cpr_drv *drv = dev;
 591         const struct cpr_desc *desc = drv->desc;
 592         irqreturn_t ret = IRQ_HANDLED;
 593         u32 val;
 594
 595         mutex_lock(&drv->lock);
 596
 597         val = cpr_read(drv, REG_RBIF_IRQ_STATUS);
 598         if (drv->flags & FLAGS_IGNORE_1ST_IRQ_STATUS)
 599                 val = cpr_read(drv, REG_RBIF_IRQ_STATUS);
 600
 601         dev_dbg(drv->dev, "IRQ_STATUS = %#02x\n", val);
 602
 603         if (!cpr_ctl_is_enabled(drv)) {
 604                 dev_dbg(drv->dev, "CPR is disabled\n");
 605                 ret = IRQ_NONE;
 606         } else if (cpr_ctl_is_busy(drv) && !desc->clamp_timer_interval) {
 607                 dev_dbg(drv->dev, "CPR measurement is not ready\n");
 608         } else if (!cpr_is_allowed(drv)) {
 609                 val = cpr_read(drv, REG_RBCPR_CTL);
 610                 dev_err_ratelimited(drv->dev,
 611                                     "Interrupt broken? RBCPR_CTL = %#02x\n",
 612                                     val);
 613                 ret = IRQ_NONE;
 614         } else {
 615                 /*
 616                  * Following sequence of handling is as per each IRQ's
 617                  * priority
 618                  */
 619                 if (val & CPR_INT_UP) {
 620                         cpr_scale(drv, UP);
 621                 } else if (val & CPR_INT_DOWN) {
 622                         cpr_scale(drv, DOWN);
 623                 } else if (val & CPR_INT_MIN) {
 624                         cpr_irq_clr_nack(drv);
 625                 } else if (val & CPR_INT_MAX) {
 626                         cpr_irq_clr_nack(drv);
 627                 } else if (val & CPR_INT_MID) {
 628                         /* RBCPR_CTL_SW_AUTO_CONT_ACK_EN is enabled */
 629                         dev_dbg(drv->dev, "IRQ occurred for Mid Flag\n");
 630                 } else {
 631                         dev_dbg(drv->dev,
 632                                 "IRQ occurred for unknown flag (%#08x)\n", val);
 633                 }
 634
 635                 /* Save register values for the corner */
 636                 cpr_corner_save(drv, drv->corner);
 637         }
 638
 639         mutex_unlock(&drv->lock);
 640
 641         return ret;
 642 }
 643
 644 static int cpr_enable(struct cpr_drv *drv)
 645 {
 646         int ret;
 647
 648         ret = regulator_enable(drv->vdd_apc);
 649         if (ret)
 650                 return ret;
 651
 652         mutex_lock(&drv->lock);
 653
 654         if (cpr_is_allowed(drv) && drv->corner) {
 655                 cpr_irq_clr(drv);
 656                 cpr_corner_restore(drv, drv->corner);
 657                 cpr_ctl_enable(drv, drv->corner);
 658         }
 659
 660         mutex_unlock(&drv->lock);
 661
 662         return 0;
 663 }
 664
 665 static int cpr_disable(struct cpr_drv *drv)
 666 {
 667         mutex_lock(&drv->lock);
 668
 669         if (cpr_is_allowed(drv)) {
 670                 cpr_ctl_disable(drv);
 671                 cpr_irq_clr(drv);
 672         }
 673
 674         mutex_unlock(&drv->lock);
 675
 676         return regulator_disable(drv->vdd_apc);
 677 }
 678
 679 static int cpr_config(struct cpr_drv *drv)
 680 {
 681         int i;
 682         u32 val, gcnt;
 683         struct corner *corner;
 684         const struct cpr_desc *desc = drv->desc;
 685
 686         /* Disable interrupt and CPR */
 687         cpr_write(drv, REG_RBIF_IRQ_EN(0), 0);
 688         cpr_write(drv, REG_RBCPR_CTL, 0);
 689
 690         /* Program the default HW ceiling, floor and vlevel */
 691         val = (RBIF_LIMIT_CEILING_DEFAULT & RBIF_LIMIT_CEILING_MASK)
 692                 << RBIF_LIMIT_CEILING_SHIFT;
 693         val |= RBIF_LIMIT_FLOOR_DEFAULT & RBIF_LIMIT_FLOOR_MASK;
 694         cpr_write(drv, REG_RBIF_LIMIT, val);
 695         cpr_write(drv, REG_RBIF_SW_VLEVEL, RBIF_SW_VLEVEL_DEFAULT);
 696
 697         /*
 698          * Clear the target quotient value and gate count of all
 699          * ring oscillators
 700          */
 701         for (i = 0; i < CPR_NUM_RING_OSC; i++)
 702                 cpr_write(drv, REG_RBCPR_GCNT_TARGET(i), 0);
 703
 704         /* Init and save gcnt */
 705         gcnt = (drv->ref_clk_khz * desc->gcnt_us) / 1000;
 706         gcnt = gcnt & RBCPR_GCNT_TARGET_GCNT_MASK;
 707         gcnt <<= RBCPR_GCNT_TARGET_GCNT_SHIFT;
 708         drv->gcnt = gcnt;
 709
 710         /* Program the delay count for the timer */
 711         val = (drv->ref_clk_khz * desc->timer_delay_us) / 1000;
 712         cpr_write(drv, REG_RBCPR_TIMER_INTERVAL, val);
 713         dev_dbg(drv->dev, "Timer count: %#0x (for %d us)\n", val,
 714                 desc->timer_delay_us);
 715
 716         /* Program Consecutive Up & Down */
 717         val = desc->timer_cons_down << RBIF_TIMER_ADJ_CONS_DOWN_SHIFT;
 718         val |= desc->timer_cons_up << RBIF_TIMER_ADJ_CONS_UP_SHIFT;
 719         val |= desc->clamp_timer_interval << RBIF_TIMER_ADJ_CLAMP_INT_SHIFT;
 720         cpr_write(drv, REG_RBIF_TIMER_ADJUST, val);
 721
 722         /* Program the control register */
 723         val = desc->up_threshold << RBCPR_CTL_UP_THRESHOLD_SHIFT;
 724         val |= desc->down_threshold << RBCPR_CTL_DN_THRESHOLD_SHIFT;
 725         val |= RBCPR_CTL_TIMER_EN | RBCPR_CTL_COUNT_MODE;
 726         val |= RBCPR_CTL_SW_AUTO_CONT_ACK_EN;
 727         cpr_write(drv, REG_RBCPR_CTL, val);
 728
 729         for (i = 0; i < drv->num_corners; i++) {
 730                 corner = &drv->corners[i];
 731                 corner->save_ctl = val;
 732                 corner->save_irq = CPR_INT_DEFAULT;
 733         }
 734
 735         cpr_irq_set(drv, CPR_INT_DEFAULT);
 736
 737         val = cpr_read(drv, REG_RBCPR_VERSION);
 738         if (val <= RBCPR_VER_2)
 739                 drv->flags |= FLAGS_IGNORE_1ST_IRQ_STATUS;
 740
 741         return 0;
 742 }
 743
 744 static int cpr_set_performance_state(struct generic_pm_domain *domain,
 745                                      unsigned int state)
 746 {
 747         struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
 748         struct corner *corner, *end;
 749         enum voltage_change_dir dir;
 750         int ret = 0, new_uV;
 751
 752         mutex_lock(&drv->lock);
 753
 754         dev_dbg(drv->dev, "%s: setting perf state: %u (prev state: %u)\n",
 755                 __func__, state, cpr_get_cur_perf_state(drv));
 756
 757         /*
 758          * Determine new corner we're going to.
 759          * Remove one since lowest performance state is 1.
 760          */
 761         corner = drv->corners + state - 1;
 762         end = &drv->corners[drv->num_corners - 1];
 763         if (corner > end || corner < drv->corners) {
 764                 ret = -EINVAL;
 765                 goto unlock;
 766         }
 767
 768         /* Determine direction */
 769         if (drv->corner > corner)
 770                 dir = DOWN;
 771         else if (drv->corner < corner)
 772                 dir = UP;
 773         else
 774                 dir = NO_CHANGE;
 775
 776         if (cpr_is_allowed(drv))
 777                 new_uV = corner->last_uV;
 778         else
 779                 new_uV = corner->uV;
 780
 781         if (cpr_is_allowed(drv))
 782                 cpr_ctl_disable(drv);
 783
 784         ret = cpr_scale_voltage(drv, corner, new_uV, dir);
 785         if (ret)
 786                 goto unlock;
 787
 788         if (cpr_is_allowed(drv)) {
 789                 cpr_irq_clr(drv);
 790                 if (drv->corner != corner)
 791                         cpr_corner_restore(drv, corner);
 792                 cpr_ctl_enable(drv, corner);
 793         }
 794
 795         drv->corner = corner;
 796
 797 unlock:
 798         mutex_unlock(&drv->lock);
 799
 800         return ret;
 801 }
 802
 803 static int
 804 cpr_populate_ring_osc_idx(struct cpr_drv *drv)
 805 {
 806         struct fuse_corner *fuse = drv->fuse_corners;
 807         struct fuse_corner *end = fuse + drv->desc->num_fuse_corners;
 808         const struct cpr_fuse *fuses = drv->cpr_fuses;
 809         u32 data;
 810         int ret;
 811
 812         for (; fuse < end; fuse++, fuses++) {
 813                 ret = nvmem_cell_read_variable_le_u32(drv->dev, fuses->ring_osc, &data);
 814                 if (ret)
 815                         return ret;
 816                 fuse->ring_osc_idx = data;
 817         }
 818
 819         return 0;
 820 }
 821
 822 static int cpr_read_fuse_uV(const struct cpr_desc *desc,
 823                             const struct fuse_corner_data *fdata,
 824                             const char *init_v_efuse,
 825                             int step_volt,
 826                             struct cpr_drv *drv)
 827 {
 828         int step_size_uV, steps, uV;
 829         u32 bits = 0;
 830         int ret;
 831
 832         ret = nvmem_cell_read_variable_le_u32(drv->dev, init_v_efuse, &bits);
 833         if (ret)
 834                 return ret;
 835
 836         steps = bits & ~BIT(desc->cpr_fuses.init_voltage_width - 1);
 837         /* Not two's complement.. instead highest bit is sign bit */
 838         if (bits & BIT(desc->cpr_fuses.init_voltage_width - 1))
 839                 steps = -steps;
 840
 841         step_size_uV = desc->cpr_fuses.init_voltage_step;
 842
 843         uV = fdata->ref_uV + steps * step_size_uV;
 844         return DIV_ROUND_UP(uV, step_volt) * step_volt;
 845 }
 846
 847 static int cpr_fuse_corner_init(struct cpr_drv *drv)
 848 {
 849         const struct cpr_desc *desc = drv->desc;
 850         const struct cpr_fuse *fuses = drv->cpr_fuses;
 851         const struct acc_desc *acc_desc = drv->acc_desc;
 852         int i;
 853         unsigned int step_volt;
 854         struct fuse_corner_data *fdata;
 855         struct fuse_corner *fuse, *end;
 856         int uV;
 857         const struct reg_sequence *accs;
 858         int ret;
 859
 860         accs = acc_desc->settings;
 861
 862         step_volt = regulator_get_linear_step(drv->vdd_apc);
 863         if (!step_volt)
 864                 return -EINVAL;
 865
 866         /* Populate fuse_corner members */
 867         fuse = drv->fuse_corners;
 868         end = &fuse[desc->num_fuse_corners - 1];
 869         fdata = desc->cpr_fuses.fuse_corner_data;
 870
 871         for (i = 0; fuse <= end; fuse++, fuses++, i++, fdata++) {
 872                 /*
 873                  * Update SoC voltages: platforms might choose a different
 874                  * regulators than the one used to characterize the algorithms
 875                  * (ie, init_voltage_step).
 876                  */
 877                 fdata->min_uV = roundup(fdata->min_uV, step_volt);
 878                 fdata->max_uV = roundup(fdata->max_uV, step_volt);
 879
 880                 /* Populate uV */
 881                 uV = cpr_read_fuse_uV(desc, fdata, fuses->init_voltage,
 882                                       step_volt, drv);
 883                 if (uV < 0)
 884                         return uV;
 885
 886                 fuse->min_uV = fdata->min_uV;
 887                 fuse->max_uV = fdata->max_uV;
 888                 fuse->uV = clamp(uV, fuse->min_uV, fuse->max_uV);
 889
 890                 if (fuse == end) {
 891                         /*
 892                          * Allow the highest fuse corner's PVS voltage to
 893                          * define the ceiling voltage for that corner in order
 894                          * to support SoC's in which variable ceiling values
 895                          * are required.
 896                          */
 897                         end->max_uV = max(end->max_uV, end->uV);
 898                 }
 899
 900                 /* Populate target quotient by scaling */
 901                 ret = nvmem_cell_read_variable_le_u32(drv->dev, fuses->quotient, &fuse->quot);
 902                 if (ret)
 903                         return ret;
 904
 905                 fuse->quot *= fdata->quot_scale;
 906                 fuse->quot += fdata->quot_offset;
 907                 fuse->quot += fdata->quot_adjust;
 908                 fuse->step_quot = desc->step_quot[fuse->ring_osc_idx];
 909
 910                 /* Populate acc settings */
 911                 fuse->accs = accs;
 912                 fuse->num_accs = acc_desc->num_regs_per_fuse;
 913                 accs += acc_desc->num_regs_per_fuse;
 914         }
 915
 916         /*
 917          * Restrict all fuse corner PVS voltages based upon per corner
 918          * ceiling and floor voltages.
 919          */
 920         for (fuse = drv->fuse_corners, i = 0; fuse <= end; fuse++, i++) {
 921                 if (fuse->uV > fuse->max_uV)
 922                         fuse->uV = fuse->max_uV;
 923                 else if (fuse->uV < fuse->min_uV)
 924                         fuse->uV = fuse->min_uV;
 925
 926                 ret = regulator_is_supported_voltage(drv->vdd_apc,
 927                                                      fuse->min_uV,
 928                                                      fuse->min_uV);
 929                 if (!ret) {
 930                         dev_err(drv->dev,
 931                                 "min uV: %d (fuse corner: %d) not supported by regulator\n",
 932                                 fuse->min_uV, i);
 933                         return -EINVAL;
 934                 }
 935
 936                 ret = regulator_is_supported_voltage(drv->vdd_apc,
 937                                                      fuse->max_uV,
 938                                                      fuse->max_uV);
 939                 if (!ret) {
 940                         dev_err(drv->dev,
 941                                 "max uV: %d (fuse corner: %d) not supported by regulator\n",
 942                                 fuse->max_uV, i);
 943                         return -EINVAL;
 944                 }
 945
 946                 dev_dbg(drv->dev,
 947                         "fuse corner %d: [%d %d %d] RO%hhu quot %d squot %d\n",
 948                         i, fuse->min_uV, fuse->uV, fuse->max_uV,
 949                         fuse->ring_osc_idx, fuse->quot, fuse->step_quot);
 950         }
 951
 952         return 0;
 953 }
 954
 955 static int cpr_calculate_scaling(const char *quot_offset,
 956                                  struct cpr_drv *drv,
 957                                  const struct fuse_corner_data *fdata,
 958                                  const struct corner *corner)
 959 {
 960         u32 quot_diff = 0;
 961         unsigned long freq_diff;
 962         int scaling;
 963         const struct fuse_corner *fuse, *prev_fuse;
 964         int ret;
 965
 966         fuse = corner->fuse_corner;
 967         prev_fuse = fuse - 1;
 968
 969         if (quot_offset) {
 970                 ret = nvmem_cell_read_variable_le_u32(drv->dev, quot_offset, &quot_diff);
 971                 if (ret)
 972                         return ret;
 973
 974                 quot_diff *= fdata->quot_offset_scale;
 975                 quot_diff += fdata->quot_offset_adjust;
 976         } else {
 977                 quot_diff = fuse->quot - prev_fuse->quot;
 978         }
 979
 980         freq_diff = fuse->max_freq - prev_fuse->max_freq;
 981         freq_diff /= 1000000; /* Convert to MHz */
 982         scaling = 1000 * quot_diff / freq_diff;
 983         return min(scaling, fdata->max_quot_scale);
 984 }
 985
 986 static int cpr_interpolate(const struct corner *corner, int step_volt,
 987                            const struct fuse_corner_data *fdata)
 988 {
 989         unsigned long f_high, f_low, f_diff;
 990         int uV_high, uV_low, uV;
 991         u64 temp, temp_limit;
 992         const struct fuse_corner *fuse, *prev_fuse;
 993
 994         fuse = corner->fuse_corner;
 995         prev_fuse = fuse - 1;
 996
 997         f_high = fuse->max_freq;
 998         f_low = prev_fuse->max_freq;
 999         uV_high = fuse->uV;
1000         uV_low = prev_fuse->uV;
1001         f_diff = fuse->max_freq - corner->freq;
1002
1003         /*
1004          * Don't interpolate in the wrong direction. This could happen
1005          * if the adjusted fuse voltage overlaps with the previous fuse's
1006          * adjusted voltage.
1007          */
1008         if (f_high <= f_low || uV_high <= uV_low || f_high <= corner->freq)
1009                 return corner->uV;
1010
1011         temp = f_diff * (uV_high - uV_low);
1012         temp = div64_ul(temp, f_high - f_low);
1013
1014         /*
1015          * max_volt_scale has units of uV/MHz while freq values
1016          * have units of Hz.  Divide by 1000000 to convert to.
1017          */
1018         temp_limit = f_diff * fdata->max_volt_scale;
1019         do_div(temp_limit, 1000000);
1020
1021         uV = uV_high - min(temp, temp_limit);
1022         return roundup(uV, step_volt);
1023 }
1024
1025 static unsigned int cpr_get_fuse_corner(struct dev_pm_opp *opp)
1026 {
1027         struct device_node *np;
1028         unsigned int fuse_corner = 0;
1029
1030         np = dev_pm_opp_get_of_node(opp);
1031         if (of_property_read_u32(np, "qcom,opp-fuse-level", &fuse_corner))
1032                 pr_err("%s: missing 'qcom,opp-fuse-level' property\n",
1033                        __func__);
1034
1035         of_node_put(np);
1036
1037         return fuse_corner;
1038 }
1039
1040 static unsigned long cpr_get_opp_hz_for_req(struct dev_pm_opp *ref,
1041                                             struct device *cpu_dev)
1042 {
1043         u64 rate = 0;
1044         struct device_node *ref_np;
1045         struct device_node *desc_np;
1046         struct device_node *child_np = NULL;
1047         struct device_node *child_req_np = NULL;
1048
1049         desc_np = dev_pm_opp_of_get_opp_desc_node(cpu_dev);
1050         if (!desc_np)
1051                 return 0;
1052
1053         ref_np = dev_pm_opp_get_of_node(ref);
1054         if (!ref_np)
1055                 goto out_ref;
1056
1057         do {
1058                 of_node_put(child_req_np);
1059                 child_np = of_get_next_available_child(desc_np, child_np);
1060                 child_req_np = of_parse_phandle(child_np, "required-opps", 0);
1061         } while (child_np && child_req_np != ref_np);
1062
1063         if (child_np && child_req_np == ref_np)
1064                 of_property_read_u64(child_np, "opp-hz", &rate);
1065
1066         of_node_put(child_req_np);
1067         of_node_put(child_np);
1068         of_node_put(ref_np);
1069 out_ref:
1070         of_node_put(desc_np);
1071
1072         return (unsigned long) rate;
1073 }
1074
1075 static int cpr_corner_init(struct cpr_drv *drv)
1076 {
1077         const struct cpr_desc *desc = drv->desc;
1078         const struct cpr_fuse *fuses = drv->cpr_fuses;
1079         int i, level, scaling = 0;
1080         unsigned int fnum, fc;
1081         const char *quot_offset;
1082         struct fuse_corner *fuse, *prev_fuse;
1083         struct corner *corner, *end;
1084         struct corner_data *cdata;
1085         const struct fuse_corner_data *fdata;
1086         bool apply_scaling;
1087         unsigned long freq_diff, freq_diff_mhz;
1088         unsigned long freq;
1089         int step_volt = regulator_get_linear_step(drv->vdd_apc);
1090         struct dev_pm_opp *opp;
1091
1092         if (!step_volt)
1093                 return -EINVAL;
1094
1095         corner = drv->corners;
1096         end = &corner[drv->num_corners - 1];
1097
1098         cdata = devm_kcalloc(drv->dev, drv->num_corners,
1099                              sizeof(struct corner_data),
1100                              GFP_KERNEL);
1101         if (!cdata)
1102                 return -ENOMEM;
1103
1104         /*
1105          * Store maximum frequency for each fuse corner based on the frequency
1106          * plan
1107          */
1108         for (level = 1; level <= drv->num_corners; level++) {
1109                 opp = dev_pm_opp_find_level_exact(&drv->pd.dev, level);
1110                 if (IS_ERR(opp))
1111                         return -EINVAL;
1112                 fc = cpr_get_fuse_corner(opp);
1113                 if (!fc) {
1114                         dev_pm_opp_put(opp);
1115                         return -EINVAL;
1116                 }
1117                 fnum = fc - 1;
1118                 freq = cpr_get_opp_hz_for_req(opp, drv->attached_cpu_dev);
1119                 if (!freq) {
1120                         dev_pm_opp_put(opp);
1121                         return -EINVAL;
1122                 }
1123                 cdata[level - 1].fuse_corner = fnum;
1124                 cdata[level - 1].freq = freq;
1125
1126                 fuse = &drv->fuse_corners[fnum];
1127                 dev_dbg(drv->dev, "freq: %lu level: %u fuse level: %u\n",
1128                         freq, dev_pm_opp_get_level(opp) - 1, fnum);
1129                 if (freq > fuse->max_freq)
1130                         fuse->max_freq = freq;
1131                 dev_pm_opp_put(opp);
1132         }
1133
1134         /*
1135          * Get the quotient adjustment scaling factor, according to:
1136          *
1137          * scaling = min(1000 * (QUOT(corner_N) - QUOT(corner_N-1))
1138          *              / (freq(corner_N) - freq(corner_N-1)), max_factor)
1139          *
1140          * QUOT(corner_N):      quotient read from fuse for fuse corner N
1141          * QUOT(corner_N-1):    quotient read from fuse for fuse corner (N - 1)
1142          * freq(corner_N):      max frequency in MHz supported by fuse corner N
1143          * freq(corner_N-1):    max frequency in MHz supported by fuse corner
1144          *                       (N - 1)
1145          *
1146          * Then walk through the corners mapped to each fuse corner
1147          * and calculate the quotient adjustment for each one using the
1148          * following formula:
1149          *
1150          * quot_adjust = (freq_max - freq_corner) * scaling / 1000
1151          *
1152          * freq_max: max frequency in MHz supported by the fuse corner
1153          * freq_corner: frequency in MHz corresponding to the corner
1154          * scaling: calculated from above equation
1155          *
1156          *
1157          *     +                           +
1158          *     |                         v |
1159          *   q |           f c           o |           f c
1160          *   u |         c               l |         c
1161          *   o |       f                 t |       f
1162          *   t |     c                   a |     c
1163          *     | c f                     g | c f
1164          *     |                         e |
1165          *     +---------------            +----------------
1166          *       0 1 2 3 4 5 6               0 1 2 3 4 5 6
1167          *          corner                      corner
1168          *
1169          *    c = corner
1170          *    f = fuse corner
1171          *
1172          */
1173         for (apply_scaling = false, i = 0; corner <= end; corner++, i++) {
1174                 fnum = cdata[i].fuse_corner;
1175                 fdata = &desc->cpr_fuses.fuse_corner_data[fnum];
1176                 quot_offset = fuses[fnum].quotient_offset;
1177                 fuse = &drv->fuse_corners[fnum];
1178                 if (fnum)
1179                         prev_fuse = &drv->fuse_corners[fnum - 1];
1180                 else
1181                         prev_fuse = NULL;
1182
1183                 corner->fuse_corner = fuse;
1184                 corner->freq = cdata[i].freq;
1185                 corner->uV = fuse->uV;
1186
1187                 if (prev_fuse && cdata[i - 1].freq == prev_fuse->max_freq) {
1188                         scaling = cpr_calculate_scaling(quot_offset, drv,
1189                                                         fdata, corner);
1190                         if (scaling < 0)
1191                                 return scaling;
1192
1193                         apply_scaling = true;
1194                 } else if (corner->freq == fuse->max_freq) {
1195                         /* This is a fuse corner; don't scale anything */
1196                         apply_scaling = false;
1197                 }
1198
1199                 if (apply_scaling) {
1200                         freq_diff = fuse->max_freq - corner->freq;
1201                         freq_diff_mhz = freq_diff / 1000000;
1202                         corner->quot_adjust = scaling * freq_diff_mhz / 1000;
1203
1204                         corner->uV = cpr_interpolate(corner, step_volt, fdata);
1205                 }
1206
1207                 corner->max_uV = fuse->max_uV;
1208                 corner->min_uV = fuse->min_uV;
1209                 corner->uV = clamp(corner->uV, corner->min_uV, corner->max_uV);
1210                 corner->last_uV = corner->uV;
1211
1212                 /* Reduce the ceiling voltage if needed */
1213                 if (desc->reduce_to_corner_uV && corner->uV < corner->max_uV)
1214                         corner->max_uV = corner->uV;
1215                 else if (desc->reduce_to_fuse_uV && fuse->uV < corner->max_uV)
1216                         corner->max_uV = max(corner->min_uV, fuse->uV);
1217
1218                 dev_dbg(drv->dev, "corner %d: [%d %d %d] quot %d\n", i,
1219                         corner->min_uV, corner->uV, corner->max_uV,
1220                         fuse->quot - corner->quot_adjust);
1221         }
1222
1223         return 0;
1224 }
1225
1226 static const struct cpr_fuse *cpr_get_fuses(struct cpr_drv *drv)
1227 {
1228         const struct cpr_desc *desc = drv->desc;
1229         struct cpr_fuse *fuses;
1230         int i;
1231
1232         fuses = devm_kcalloc(drv->dev, desc->num_fuse_corners,
1233                              sizeof(struct cpr_fuse),
1234                              GFP_KERNEL);
1235         if (!fuses)
1236                 return ERR_PTR(-ENOMEM);
1237
1238         for (i = 0; i < desc->num_fuse_corners; i++) {
1239                 char tbuf[32];
1240
1241                 snprintf(tbuf, 32, "cpr_ring_osc%d", i + 1);
1242                 fuses[i].ring_osc = devm_kstrdup(drv->dev, tbuf, GFP_KERNEL);
1243                 if (!fuses[i].ring_osc)
1244                         return ERR_PTR(-ENOMEM);
1245
1246                 snprintf(tbuf, 32, "cpr_init_voltage%d", i + 1);
1247                 fuses[i].init_voltage = devm_kstrdup(drv->dev, tbuf,
1248                                                      GFP_KERNEL);
1249                 if (!fuses[i].init_voltage)
1250                         return ERR_PTR(-ENOMEM);
1251
1252                 snprintf(tbuf, 32, "cpr_quotient%d", i + 1);
1253                 fuses[i].quotient = devm_kstrdup(drv->dev, tbuf, GFP_KERNEL);
1254                 if (!fuses[i].quotient)
1255                         return ERR_PTR(-ENOMEM);
1256
1257                 snprintf(tbuf, 32, "cpr_quotient_offset%d", i + 1);
1258                 fuses[i].quotient_offset = devm_kstrdup(drv->dev, tbuf,
1259                                                         GFP_KERNEL);
1260                 if (!fuses[i].quotient_offset)
1261                         return ERR_PTR(-ENOMEM);
1262         }
1263
1264         return fuses;
1265 }
1266
1267 static void cpr_set_loop_allowed(struct cpr_drv *drv)
1268 {
1269         drv->loop_disabled = false;
1270 }
1271
1272 static int cpr_init_parameters(struct cpr_drv *drv)
1273 {
1274         const struct cpr_desc *desc = drv->desc;
1275         struct clk *clk;
1276
1277         clk = clk_get(drv->dev, "ref");
1278         if (IS_ERR(clk))
1279                 return PTR_ERR(clk);
1280
1281         drv->ref_clk_khz = clk_get_rate(clk) / 1000;
1282         clk_put(clk);
1283
1284         if (desc->timer_cons_up > RBIF_TIMER_ADJ_CONS_UP_MASK ||
1285             desc->timer_cons_down > RBIF_TIMER_ADJ_CONS_DOWN_MASK ||
1286             desc->up_threshold > RBCPR_CTL_UP_THRESHOLD_MASK ||
1287             desc->down_threshold > RBCPR_CTL_DN_THRESHOLD_MASK ||
1288             desc->idle_clocks > RBCPR_STEP_QUOT_IDLE_CLK_MASK ||
1289             desc->clamp_timer_interval > RBIF_TIMER_ADJ_CLAMP_INT_MASK)
1290                 return -EINVAL;
1291
1292         dev_dbg(drv->dev, "up threshold = %u, down threshold = %u\n",
1293                 desc->up_threshold, desc->down_threshold);
1294
1295         return 0;
1296 }
1297
1298 static int cpr_find_initial_corner(struct cpr_drv *drv)
1299 {
1300         unsigned long rate;
1301         const struct corner *end;
1302         struct corner *iter;
1303         unsigned int i = 0;
1304
1305         if (!drv->cpu_clk) {
1306                 dev_err(drv->dev, "cannot get rate from NULL clk\n");
1307                 return -EINVAL;
1308         }
1309
1310         end = &drv->corners[drv->num_corners - 1];
1311         rate = clk_get_rate(drv->cpu_clk);
1312
1313         /*
1314          * Some bootloaders set a CPU clock frequency that is not defined
1315          * in the OPP table. When running at an unlisted frequency,
1316          * cpufreq_online() will change to the OPP which has the lowest
1317          * frequency, at or above the unlisted frequency.
1318          * Since cpufreq_online() always "rounds up" in the case of an
1319          * unlisted frequency, this function always "rounds down" in case
1320          * of an unlisted frequency. That way, when cpufreq_online()
1321          * triggers the first ever call to cpr_set_performance_state(),
1322          * it will correctly determine the direction as UP.
1323          */
1324         for (iter = drv->corners; iter <= end; iter++) {
1325                 if (iter->freq > rate)
1326                         break;
1327                 i++;
1328                 if (iter->freq == rate) {
1329                         drv->corner = iter;
1330                         break;
1331                 }
1332                 if (iter->freq < rate)
1333                         drv->corner = iter;
1334         }
1335
1336         if (!drv->corner) {
1337                 dev_err(drv->dev, "boot up corner not found\n");
1338                 return -EINVAL;
1339         }
1340
1341         dev_dbg(drv->dev, "boot up perf state: %u\n", i);
1342
1343         return 0;
1344 }
1345
1346 static const struct cpr_desc qcs404_cpr_desc = {
1347         .num_fuse_corners = 3,
1348         .min_diff_quot = CPR_FUSE_MIN_QUOT_DIFF,
1349         .step_quot = (int []){ 25, 25, 25, },
1350         .timer_delay_us = 5000,
1351         .timer_cons_up = 0,
1352         .timer_cons_down = 2,
1353         .up_threshold = 1,
1354         .down_threshold = 3,
1355         .idle_clocks = 15,
1356         .gcnt_us = 1,
1357         .vdd_apc_step_up_limit = 1,
1358         .vdd_apc_step_down_limit = 1,
1359         .cpr_fuses = {
1360                 .init_voltage_step = 8000,
1361                 .init_voltage_width = 6,
1362                 .fuse_corner_data = (struct fuse_corner_data[]){
1363                         /* fuse corner 0 */
1364                         {
1365                                 .ref_uV = 1224000,
1366                                 .max_uV = 1224000,
1367                                 .min_uV = 1048000,
1368                                 .max_volt_scale = 0,
1369                                 .max_quot_scale = 0,
1370                                 .quot_offset = 0,
1371                                 .quot_scale = 1,
1372                                 .quot_adjust = 0,
1373                                 .quot_offset_scale = 5,
1374                                 .quot_offset_adjust = 0,
1375                         },
1376                         /* fuse corner 1 */
1377                         {
1378                                 .ref_uV = 1288000,
1379                                 .max_uV = 1288000,
1380                                 .min_uV = 1048000,
1381                                 .max_volt_scale = 2000,
1382                                 .max_quot_scale = 1400,
1383                                 .quot_offset = 0,
1384                                 .quot_scale = 1,
1385                                 .quot_adjust = -20,
1386                                 .quot_offset_scale = 5,
1387                                 .quot_offset_adjust = 0,
1388                         },
1389                         /* fuse corner 2 */
1390                         {
1391                                 .ref_uV = 1352000,
1392                                 .max_uV = 1384000,
1393                                 .min_uV = 1088000,
1394                                 .max_volt_scale = 2000,
1395                                 .max_quot_scale = 1400,
1396                                 .quot_offset = 0,
1397                                 .quot_scale = 1,
1398                                 .quot_adjust = 0,
1399                                 .quot_offset_scale = 5,
1400                                 .quot_offset_adjust = 0,
1401                         },
1402                 },
1403         },
1404 };
1405
1406 static const struct acc_desc qcs404_acc_desc = {
1407         .settings = (struct reg_sequence[]){
1408                 { 0xb120, 0x1041040 },
1409                 { 0xb124, 0x41 },
1410                 { 0xb120, 0x0 },
1411                 { 0xb124, 0x0 },
1412                 { 0xb120, 0x0 },
1413                 { 0xb124, 0x0 },
1414         },
1415         .config = (struct reg_sequence[]){
1416                 { 0xb138, 0xff },
1417                 { 0xb130, 0x5555 },
1418         },
1419         .num_regs_per_fuse = 2,
1420 };
1421
1422 static const struct cpr_acc_desc qcs404_cpr_acc_desc = {
1423         .cpr_desc = &qcs404_cpr_desc,
1424         .acc_desc = &qcs404_acc_desc,
1425 };
1426
1427 static int cpr_power_off(struct generic_pm_domain *domain)
1428 {
1429         struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1430
1431         return cpr_disable(drv);
1432 }
1433
1434 static int cpr_power_on(struct generic_pm_domain *domain)
1435 {
1436         struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1437
1438         return cpr_enable(drv);
1439 }
1440
1441 static int cpr_pd_attach_dev(struct generic_pm_domain *domain,
1442                              struct device *dev)
1443 {
1444         struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1445         const struct acc_desc *acc_desc = drv->acc_desc;
1446         int ret = 0;
1447
1448         mutex_lock(&drv->lock);
1449
1450         dev_dbg(drv->dev, "attach callback for: %s\n", dev_name(dev));
1451
1452         /*
1453          * This driver only supports scaling voltage for a CPU cluster
1454          * where all CPUs in the cluster share a single regulator.
1455          * Therefore, save the struct device pointer only for the first
1456          * CPU device that gets attached. There is no need to do any
1457          * additional initialization when further CPUs get attached.
1458          */
1459         if (drv->attached_cpu_dev)
1460                 goto unlock;
1461
1462         /*
1463          * cpr_scale_voltage() requires the direction (if we are changing
1464          * to a higher or lower OPP). The first time
1465          * cpr_set_performance_state() is called, there is no previous
1466          * performance state defined. Therefore, we call
1467          * cpr_find_initial_corner() that gets the CPU clock frequency
1468          * set by the bootloader, so that we can determine the direction
1469          * the first time cpr_set_performance_state() is called.
1470          */
1471         drv->cpu_clk = devm_clk_get(dev, NULL);
1472         if (IS_ERR(drv->cpu_clk)) {
1473                 ret = PTR_ERR(drv->cpu_clk);
1474                 if (ret != -EPROBE_DEFER)
1475                         dev_err(drv->dev, "could not get cpu clk: %d\n", ret);
1476                 goto unlock;
1477         }
1478         drv->attached_cpu_dev = dev;
1479
1480         dev_dbg(drv->dev, "using cpu clk from: %s\n",
1481                 dev_name(drv->attached_cpu_dev));
1482
1483         /*
1484          * Everything related to (virtual) corners has to be initialized
1485          * here, when attaching to the power domain, since we need to know
1486          * the maximum frequency for each fuse corner, and this is only
1487          * available after the cpufreq driver has attached to us.
1488          * The reason for this is that we need to know the highest
1489          * frequency associated with each fuse corner.
1490          */
1491         ret = dev_pm_opp_get_opp_count(&drv->pd.dev);
1492         if (ret < 0) {
1493                 dev_err(drv->dev, "could not get OPP count\n");
1494                 goto unlock;
1495         }
1496         drv->num_corners = ret;
1497
1498         if (drv->num_corners < 2) {
1499                 dev_err(drv->dev, "need at least 2 OPPs to use CPR\n");
1500                 ret = -EINVAL;
1501                 goto unlock;
1502         }
1503
1504         drv->corners = devm_kcalloc(drv->dev, drv->num_corners,
1505                                     sizeof(*drv->corners),
1506                                     GFP_KERNEL);
1507         if (!drv->corners) {
1508                 ret = -ENOMEM;
1509                 goto unlock;
1510         }
1511
1512         ret = cpr_corner_init(drv);
1513         if (ret)
1514                 goto unlock;
1515
1516         cpr_set_loop_allowed(drv);
1517
1518         ret = cpr_init_parameters(drv);
1519         if (ret)
1520                 goto unlock;
1521
1522         /* Configure CPR HW but keep it disabled */
1523         ret = cpr_config(drv);
1524         if (ret)
1525                 goto unlock;
1526
1527         ret = cpr_find_initial_corner(drv);
1528         if (ret)
1529                 goto unlock;
1530
1531         if (acc_desc->config)
1532                 regmap_multi_reg_write(drv->tcsr, acc_desc->config,
1533                                        acc_desc->num_regs_per_fuse);
1534
1535         /* Enable ACC if required */
1536         if (acc_desc->enable_mask)
1537                 regmap_update_bits(drv->tcsr, acc_desc->enable_reg,
1538                                    acc_desc->enable_mask,
1539                                    acc_desc->enable_mask);
1540
1541         dev_info(drv->dev, "driver initialized with %u OPPs\n",
1542                  drv->num_corners);
1543
1544 unlock:
1545         mutex_unlock(&drv->lock);
1546
1547         return ret;
1548 }
1549
1550 static int cpr_debug_info_show(struct seq_file *s, void *unused)
1551 {
1552         u32 gcnt, ro_sel, ctl, irq_status, reg, error_steps;
1553         u32 step_dn, step_up, error, error_lt0, busy;
1554         struct cpr_drv *drv = s->private;
1555         struct fuse_corner *fuse_corner;
1556         struct corner *corner;
1557
1558         corner = drv->corner;
1559         fuse_corner = corner->fuse_corner;
1560
1561         seq_printf(s, "corner, current_volt = %d uV\n",
1562                        corner->last_uV);
1563
1564         ro_sel = fuse_corner->ring_osc_idx;
1565         gcnt = cpr_read(drv, REG_RBCPR_GCNT_TARGET(ro_sel));
1566         seq_printf(s, "rbcpr_gcnt_target (%u) = %#02X\n", ro_sel, gcnt);
1567
1568         ctl = cpr_read(drv, REG_RBCPR_CTL);
1569         seq_printf(s, "rbcpr_ctl = %#02X\n", ctl);
1570
1571         irq_status = cpr_read(drv, REG_RBIF_IRQ_STATUS);
1572         seq_printf(s, "rbcpr_irq_status = %#02X\n", irq_status);
1573
1574         reg = cpr_read(drv, REG_RBCPR_RESULT_0);
1575         seq_printf(s, "rbcpr_result_0 = %#02X\n", reg);
1576
1577         step_dn = reg & 0x01;
1578         step_up = (reg >> RBCPR_RESULT0_STEP_UP_SHIFT) & 0x01;
1579         seq_printf(s, "  [step_dn = %u", step_dn);
1580
1581         seq_printf(s, ", step_up = %u", step_up);
1582
1583         error_steps = (reg >> RBCPR_RESULT0_ERROR_STEPS_SHIFT)
1584                                 & RBCPR_RESULT0_ERROR_STEPS_MASK;
1585         seq_printf(s, ", error_steps = %u", error_steps);
1586
1587         error = (reg >> RBCPR_RESULT0_ERROR_SHIFT) & RBCPR_RESULT0_ERROR_MASK;
1588         seq_printf(s, ", error = %u", error);
1589
1590         error_lt0 = (reg >> RBCPR_RESULT0_ERROR_LT0_SHIFT) & 0x01;
1591         seq_printf(s, ", error_lt_0 = %u", error_lt0);
1592
1593         busy = (reg >> RBCPR_RESULT0_BUSY_SHIFT) & 0x01;
1594         seq_printf(s, ", busy = %u]\n", busy);
1595
1596         return 0;
1597 }
1598 DEFINE_SHOW_ATTRIBUTE(cpr_debug_info);
1599
1600 static void cpr_debugfs_init(struct cpr_drv *drv)
1601 {
1602         drv->debugfs = debugfs_create_dir("qcom_cpr", NULL);
1603
1604         debugfs_create_file("debug_info", 0444, drv->debugfs,
1605                             drv, &cpr_debug_info_fops);
1606 }
1607
1608 static int cpr_probe(struct platform_device *pdev)
1609 {
1610         struct device *dev = &pdev->dev;
1611         struct cpr_drv *drv;
1612         int irq, ret;
1613         const struct cpr_acc_desc *data;
1614         struct device_node *np;
1615         u32 cpr_rev = FUSE_REVISION_UNKNOWN;
1616
1617         data = of_device_get_match_data(dev);
1618         if (!data || !data->cpr_desc || !data->acc_desc)
1619                 return -EINVAL;
1620
1621         drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);
1622         if (!drv)
1623                 return -ENOMEM;
1624         drv->dev = dev;
1625         drv->desc = data->cpr_desc;
1626         drv->acc_desc = data->acc_desc;
1627
1628         drv->fuse_corners = devm_kcalloc(dev, drv->desc->num_fuse_corners,
1629                                          sizeof(*drv->fuse_corners),
1630                                          GFP_KERNEL);
1631         if (!drv->fuse_corners)
1632                 return -ENOMEM;
1633
1634         np = of_parse_phandle(dev->of_node, "acc-syscon", 0);
1635         if (!np)
1636                 return -ENODEV;
1637
1638         drv->tcsr = syscon_node_to_regmap(np);
1639         of_node_put(np);
1640         if (IS_ERR(drv->tcsr))
1641                 return PTR_ERR(drv->tcsr);
1642
1643         drv->base = devm_platform_ioremap_resource(pdev, 0);
1644         if (IS_ERR(drv->base))
1645                 return PTR_ERR(drv->base);
1646
1647         irq = platform_get_irq(pdev, 0);
1648         if (irq < 0)
1649                 return -EINVAL;
1650
1651         drv->vdd_apc = devm_regulator_get(dev, "vdd-apc");
1652         if (IS_ERR(drv->vdd_apc))
1653                 return PTR_ERR(drv->vdd_apc);
1654
1655         /*
1656          * Initialize fuse corners, since it simply depends
1657          * on data in efuses.
1658          * Everything related to (virtual) corners has to be
1659          * initialized after attaching to the power domain,
1660          * since it depends on the CPU's OPP table.
1661          */
1662         ret = nvmem_cell_read_variable_le_u32(dev, "cpr_fuse_revision", &cpr_rev);
1663         if (ret)
1664                 return ret;
1665
1666         drv->cpr_fuses = cpr_get_fuses(drv);
1667         if (IS_ERR(drv->cpr_fuses))
1668                 return PTR_ERR(drv->cpr_fuses);
1669
1670         ret = cpr_populate_ring_osc_idx(drv);
1671         if (ret)
1672                 return ret;
1673
1674         ret = cpr_fuse_corner_init(drv);
1675         if (ret)
1676                 return ret;
1677
1678         mutex_init(&drv->lock);
1679
1680         ret = devm_request_threaded_irq(dev, irq, NULL,
1681                                         cpr_irq_handler,
1682                                         IRQF_ONESHOT | IRQF_TRIGGER_RISING,
1683                                         "cpr", drv);
1684         if (ret)
1685                 return ret;
1686
1687         drv->pd.name = devm_kstrdup_const(dev, dev->of_node->full_name,
1688                                           GFP_KERNEL);
1689         if (!drv->pd.name)
1690                 return -EINVAL;
1691
1692         drv->pd.power_off = cpr_power_off;
1693         drv->pd.power_on = cpr_power_on;
1694         drv->pd.set_performance_state = cpr_set_performance_state;
1695         drv->pd.attach_dev = cpr_pd_attach_dev;
1696
1697         ret = pm_genpd_init(&drv->pd, NULL, true);
1698         if (ret)
1699                 return ret;
1700
1701         ret = of_genpd_add_provider_simple(dev->of_node, &drv->pd);
1702         if (ret)
1703                 goto err_remove_genpd;
1704
1705         platform_set_drvdata(pdev, drv);
1706         cpr_debugfs_init(drv);
1707
1708         return 0;
1709
1710 err_remove_genpd:
1711         pm_genpd_remove(&drv->pd);
1712         return ret;
1713 }
1714
1715 static void cpr_remove(struct platform_device *pdev)
1716 {
1717         struct cpr_drv *drv = platform_get_drvdata(pdev);
1718
1719         if (cpr_is_allowed(drv)) {
1720                 cpr_ctl_disable(drv);
1721                 cpr_irq_set(drv, 0);
1722         }
1723
1724         of_genpd_del_provider(pdev->dev.of_node);
1725         pm_genpd_remove(&drv->pd);
1726
1727         debugfs_remove_recursive(drv->debugfs);
1728 }
1729
1730 static const struct of_device_id cpr_match_table[] = {
1731         { .compatible = "qcom,qcs404-cpr", .data = &qcs404_cpr_acc_desc },
1732         { }
1733 };
1734 MODULE_DEVICE_TABLE(of, cpr_match_table);
1735
1736 static struct platform_driver cpr_driver = {
1737         .probe          = cpr_probe,
1738         .remove_new     = cpr_remove,
1739         .driver         = {
1740                 .name   = "qcom-cpr",
1741                 .of_match_table = cpr_match_table,
1742         },
1743 };
1744 module_platform_driver(cpr_driver);
1745
1746 MODULE_DESCRIPTION("Core Power Reduction (CPR) driver");
1747 MODULE_LICENSE("GPL v2");