drivers/memory/samsung/exynos5422-dmc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019 Samsung Electronics Co., Ltd.
   4  * Author: Lukasz Luba <l.luba@partner.samsung.com>
   5  */
   6
   7 #include <linux/clk.h>
   8 #include <linux/devfreq.h>
   9 #include <linux/devfreq-event.h>
  10 #include <linux/device.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/io.h>
  13 #include <linux/mfd/syscon.h>
  14 #include <linux/module.h>
  15 #include <linux/moduleparam.h>
  16 #include <linux/of_device.h>
  17 #include <linux/pm_opp.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/regmap.h>
  20 #include <linux/regulator/consumer.h>
  21 #include <linux/slab.h>
  22 #include "../jedec_ddr.h"
  23 #include "../of_memory.h"
  24
  25 static int irqmode;
  26 module_param(irqmode, int, 0644);
  27 MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)");
  28
  29 #define EXYNOS5_DREXI_TIMINGAREF                (0x0030)
  30 #define EXYNOS5_DREXI_TIMINGROW0                (0x0034)
  31 #define EXYNOS5_DREXI_TIMINGDATA0               (0x0038)
  32 #define EXYNOS5_DREXI_TIMINGPOWER0              (0x003C)
  33 #define EXYNOS5_DREXI_TIMINGROW1                (0x00E4)
  34 #define EXYNOS5_DREXI_TIMINGDATA1               (0x00E8)
  35 #define EXYNOS5_DREXI_TIMINGPOWER1              (0x00EC)
  36 #define CDREX_PAUSE                             (0x2091c)
  37 #define CDREX_LPDDR3PHY_CON3                    (0x20a20)
  38 #define CDREX_LPDDR3PHY_CLKM_SRC                (0x20700)
  39 #define EXYNOS5_TIMING_SET_SWI                  BIT(28)
  40 #define USE_MX_MSPLL_TIMINGS                    (1)
  41 #define USE_BPLL_TIMINGS                        (0)
  42 #define EXYNOS5_AREF_NORMAL                     (0x2e)
  43
  44 #define DREX_PPCCLKCON          (0x0130)
  45 #define DREX_PEREV2CONFIG       (0x013c)
  46 #define DREX_PMNC_PPC           (0xE000)
  47 #define DREX_CNTENS_PPC         (0xE010)
  48 #define DREX_CNTENC_PPC         (0xE020)
  49 #define DREX_INTENS_PPC         (0xE030)
  50 #define DREX_INTENC_PPC         (0xE040)
  51 #define DREX_FLAG_PPC           (0xE050)
  52 #define DREX_PMCNT2_PPC         (0xE130)
  53
  54 /*
  55  * A value for register DREX_PMNC_PPC which should be written to reset
  56  * the cycle counter CCNT (a reference wall clock). It sets zero to the
  57  * CCNT counter.
  58  */
  59 #define CC_RESET                BIT(2)
  60
  61 /*
  62  * A value for register DREX_PMNC_PPC which does the reset of all performance
  63  * counters to zero.
  64  */
  65 #define PPC_COUNTER_RESET       BIT(1)
  66
  67 /*
  68  * Enables all configured counters (including cycle counter). The value should
  69  * be written to the register DREX_PMNC_PPC.
  70  */
  71 #define PPC_ENABLE              BIT(0)
  72
  73 /* A value for register DREX_PPCCLKCON which enables performance events clock.
  74  * Must be written before first access to the performance counters register
  75  * set, otherwise it could crash.
  76  */
  77 #define PEREV_CLK_EN            BIT(0)
  78
  79 /*
  80  * Values which are used to enable counters, interrupts or configure flags of
  81  * the performance counters. They configure counter 2 and cycle counter.
  82  */
  83 #define PERF_CNT2               BIT(2)
  84 #define PERF_CCNT               BIT(31)
  85
  86 /*
  87  * Performance event types which are used for setting the preferred event
  88  * to track in the counters.
  89  * There is a set of different types, the values are from range 0 to 0x6f.
  90  * These settings should be written to the configuration register which manages
  91  * the type of the event (register DREX_PEREV2CONFIG).
  92  */
  93 #define READ_TRANSFER_CH0       (0x6d)
  94 #define READ_TRANSFER_CH1       (0x6f)
  95
  96 #define PERF_COUNTER_START_VALUE 0xff000000
  97 #define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL
  98
  99 /**
 100  * struct dmc_opp_table - Operating level desciption
 101  * @freq_hz:            target frequency in Hz
 102  * @volt_uv:            target voltage in uV
 103  *
 104  * Covers frequency and voltage settings of the DMC operating mode.
 105  */
 106 struct dmc_opp_table {
 107         u32 freq_hz;
 108         u32 volt_uv;
 109 };
 110
 111 /**
 112  * struct exynos5_dmc - main structure describing DMC device
 113  * @dev:                DMC device
 114  * @df:                 devfreq device structure returned by devfreq framework
 115  * @gov_data:           configuration of devfreq governor
 116  * @base_drexi0:        DREX0 registers mapping
 117  * @base_drexi1:        DREX1 registers mapping
 118  * @clk_regmap:         regmap for clock controller registers
 119  * @lock:               protects curr_rate and frequency/voltage setting section
 120  * @curr_rate:          current frequency
 121  * @curr_volt:          current voltage
 122  * @opp:                OPP table
 123  * @opp_count:          number of 'opp' elements
 124  * @timings_arr_size:   number of 'timings' elements
 125  * @timing_row:         values for timing row register, for each OPP
 126  * @timing_data:        values for timing data register, for each OPP
 127  * @timing_power:       balues for timing power register, for each OPP
 128  * @timings:            DDR memory timings, from device tree
 129  * @min_tck:            DDR memory minimum timing values, from device tree
 130  * @bypass_timing_row:  value for timing row register for bypass timings
 131  * @bypass_timing_data: value for timing data register for bypass timings
 132  * @bypass_timing_power:        value for timing power register for bypass
 133  *                              timings
 134  * @vdd_mif:            Memory interface regulator
 135  * @fout_spll:          clock: SPLL
 136  * @fout_bpll:          clock: BPLL
 137  * @mout_spll:          clock: mux SPLL
 138  * @mout_bpll:          clock: mux BPLL
 139  * @mout_mclk_cdrex:    clock: mux mclk_cdrex
 140  * @mout_mx_mspll_ccore:        clock: mux mx_mspll_ccore
 141  * @counter:            devfreq events
 142  * @num_counters:       number of 'counter' elements
 143  * @last_overflow_ts:   time (in ns) of last overflow of each DREX
 144  * @load:               utilization in percents
 145  * @total:              total time between devfreq events
 146  * @in_irq_mode:        whether running in interrupt mode (true)
 147  *                      or polling (false)
 148  *
 149  * The main structure for the Dynamic Memory Controller which covers clocks,
 150  * memory regions, HW information, parameters and current operating mode.
 151  */
 152 struct exynos5_dmc {
 153         struct device *dev;
 154         struct devfreq *df;
 155         struct devfreq_simple_ondemand_data gov_data;
 156         void __iomem *base_drexi0;
 157         void __iomem *base_drexi1;
 158         struct regmap *clk_regmap;
 159         /* Protects curr_rate and frequency/voltage setting section */
 160         struct mutex lock;
 161         unsigned long curr_rate;
 162         unsigned long curr_volt;
 163         struct dmc_opp_table *opp;
 164         int opp_count;
 165         u32 timings_arr_size;
 166         u32 *timing_row;
 167         u32 *timing_data;
 168         u32 *timing_power;
 169         const struct lpddr3_timings *timings;
 170         const struct lpddr3_min_tck *min_tck;
 171         u32 bypass_timing_row;
 172         u32 bypass_timing_data;
 173         u32 bypass_timing_power;
 174         struct regulator *vdd_mif;
 175         struct clk *fout_spll;
 176         struct clk *fout_bpll;
 177         struct clk *mout_spll;
 178         struct clk *mout_bpll;
 179         struct clk *mout_mclk_cdrex;
 180         struct clk *mout_mx_mspll_ccore;
 181         struct devfreq_event_dev **counter;
 182         int num_counters;
 183         u64 last_overflow_ts[2];
 184         unsigned long load;
 185         unsigned long total;
 186         bool in_irq_mode;
 187 };
 188
 189 #define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \
 190         { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }
 191
 192 #define TIMING_VAL2REG(timing, t_val)                   \
 193 ({                                                      \
 194                 u32 __val;                              \
 195                 __val = (t_val) << (timing)->bit_beg;   \
 196                 __val;                                  \
 197 })
 198
 199 struct timing_reg {
 200         char *name;
 201         int bit_beg;
 202         int bit_end;
 203         unsigned int val;
 204 };
 205
 206 static const struct timing_reg timing_row_reg_fields[] = {
 207         TIMING_FIELD("tRFC", 24, 31),
 208         TIMING_FIELD("tRRD", 20, 23),
 209         TIMING_FIELD("tRP", 16, 19),
 210         TIMING_FIELD("tRCD", 12, 15),
 211         TIMING_FIELD("tRC", 6, 11),
 212         TIMING_FIELD("tRAS", 0, 5),
 213 };
 214
 215 static const struct timing_reg timing_data_reg_fields[] = {
 216         TIMING_FIELD("tWTR", 28, 31),
 217         TIMING_FIELD("tWR", 24, 27),
 218         TIMING_FIELD("tRTP", 20, 23),
 219         TIMING_FIELD("tW2W-C2C", 14, 14),
 220         TIMING_FIELD("tR2R-C2C", 12, 12),
 221         TIMING_FIELD("WL", 8, 11),
 222         TIMING_FIELD("tDQSCK", 4, 7),
 223         TIMING_FIELD("RL", 0, 3),
 224 };
 225
 226 static const struct timing_reg timing_power_reg_fields[] = {
 227         TIMING_FIELD("tFAW", 26, 31),
 228         TIMING_FIELD("tXSR", 16, 25),
 229         TIMING_FIELD("tXP", 8, 15),
 230         TIMING_FIELD("tCKE", 4, 7),
 231         TIMING_FIELD("tMRD", 0, 3),
 232 };
 233
 234 #define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \
 235                       ARRAY_SIZE(timing_data_reg_fields) + \
 236                       ARRAY_SIZE(timing_power_reg_fields))
 237
 238 static int exynos5_counters_set_event(struct exynos5_dmc *dmc)
 239 {
 240         int i, ret;
 241
 242         for (i = 0; i < dmc->num_counters; i++) {
 243                 if (!dmc->counter[i])
 244                         continue;
 245                 ret = devfreq_event_set_event(dmc->counter[i]);
 246                 if (ret < 0)
 247                         return ret;
 248         }
 249         return 0;
 250 }
 251
 252 static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)
 253 {
 254         int i, ret;
 255
 256         for (i = 0; i < dmc->num_counters; i++) {
 257                 if (!dmc->counter[i])
 258                         continue;
 259                 ret = devfreq_event_enable_edev(dmc->counter[i]);
 260                 if (ret < 0)
 261                         return ret;
 262         }
 263         return 0;
 264 }
 265
 266 static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)
 267 {
 268         int i, ret;
 269
 270         for (i = 0; i < dmc->num_counters; i++) {
 271                 if (!dmc->counter[i])
 272                         continue;
 273                 ret = devfreq_event_disable_edev(dmc->counter[i]);
 274                 if (ret < 0)
 275                         return ret;
 276         }
 277         return 0;
 278 }
 279
 280 /**
 281  * find_target_freq_id() - Finds requested frequency in local DMC configuration
 282  * @dmc:        device for which the information is checked
 283  * @target_rate:        requested frequency in KHz
 284  *
 285  * Seeks in the local DMC driver structure for the requested frequency value
 286  * and returns index or error value.
 287  */
 288 static int find_target_freq_idx(struct exynos5_dmc *dmc,
 289                                 unsigned long target_rate)
 290 {
 291         int i;
 292
 293         for (i = dmc->opp_count - 1; i >= 0; i--)
 294                 if (dmc->opp[i].freq_hz <= target_rate)
 295                         return i;
 296
 297         return -EINVAL;
 298 }
 299
 300 /**
 301  * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings
 302  * @dmc:        device for which the new settings is going to be applied
 303  * @set:        boolean variable passing set value
 304  *
 305  * Changes the register set, which holds timing parameters.
 306  * There is two register sets: 0 and 1. The register set 0
 307  * is used in normal operation when the clock is provided from main PLL.
 308  * The bank register set 1 is used when the main PLL frequency is going to be
 309  * changed and the clock is taken from alternative, stable source.
 310  * This function switches between these banks according to the
 311  * currently used clock source.
 312  */
 313 static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)
 314 {
 315         unsigned int reg;
 316         int ret;
 317
 318         ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, &reg);
 319         if (ret)
 320                 return ret;
 321
 322         if (set)
 323                 reg |= EXYNOS5_TIMING_SET_SWI;
 324         else
 325                 reg &= ~EXYNOS5_TIMING_SET_SWI;
 326
 327         regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);
 328
 329         return 0;
 330 }
 331
 332 /**
 333  * exynos5_init_freq_table() - Initialized PM OPP framework
 334  * @dmc:        DMC device for which the frequencies are used for OPP init
 335  * @profile:    devfreq device's profile
 336  *
 337  * Populate the devfreq device's OPP table based on current frequency, voltage.
 338  */
 339 static int exynos5_init_freq_table(struct exynos5_dmc *dmc,
 340                                    struct devfreq_dev_profile *profile)
 341 {
 342         int i, ret;
 343         int idx;
 344         unsigned long freq;
 345
 346         ret = dev_pm_opp_of_add_table(dmc->dev);
 347         if (ret < 0) {
 348                 dev_err(dmc->dev, "Failed to get OPP table\n");
 349                 return ret;
 350         }
 351
 352         dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev);
 353
 354         dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count,
 355                                       sizeof(struct dmc_opp_table), GFP_KERNEL);
 356         if (!dmc->opp)
 357                 goto err_opp;
 358
 359         idx = dmc->opp_count - 1;
 360         for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {
 361                 struct dev_pm_opp *opp;
 362
 363                 opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq);
 364                 if (IS_ERR(opp))
 365                         goto err_opp;
 366
 367                 dmc->opp[idx - i].freq_hz = freq;
 368                 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);
 369
 370                 dev_pm_opp_put(opp);
 371         }
 372
 373         return 0;
 374
 375 err_opp:
 376         dev_pm_opp_of_remove_table(dmc->dev);
 377
 378         return -EINVAL;
 379 }
 380
 381 /**
 382  * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings
 383  * @dmc:        device for which the new settings is going to be applied
 384  *
 385  * Low-level function for changing timings for DRAM memory clocking from
 386  * 'bypass' clock source (fixed frequency @400MHz).
 387  * It uses timing bank registers set 1.
 388  */
 389 static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)
 390 {
 391         writel(EXYNOS5_AREF_NORMAL,
 392                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
 393
 394         writel(dmc->bypass_timing_row,
 395                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);
 396         writel(dmc->bypass_timing_row,
 397                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);
 398         writel(dmc->bypass_timing_data,
 399                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);
 400         writel(dmc->bypass_timing_data,
 401                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);
 402         writel(dmc->bypass_timing_power,
 403                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);
 404         writel(dmc->bypass_timing_power,
 405                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);
 406 }
 407
 408 /**
 409  * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings
 410  * @dmc:        device for which the new settings is going to be applied
 411  * @target_rate:        target frequency of the DMC
 412  *
 413  * Low-level function for changing timings for DRAM memory operating from main
 414  * clock source (BPLL), which can have different frequencies. Thus, each
 415  * frequency must have corresponding timings register values in order to keep
 416  * the needed delays.
 417  * It uses timing bank registers set 0.
 418  */
 419 static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,
 420                                        unsigned long target_rate)
 421 {
 422         int idx;
 423
 424         for (idx = dmc->opp_count - 1; idx >= 0; idx--)
 425                 if (dmc->opp[idx].freq_hz <= target_rate)
 426                         break;
 427
 428         if (idx < 0)
 429                 return -EINVAL;
 430
 431         writel(EXYNOS5_AREF_NORMAL,
 432                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
 433
 434         writel(dmc->timing_row[idx],
 435                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);
 436         writel(dmc->timing_row[idx],
 437                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);
 438         writel(dmc->timing_data[idx],
 439                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);
 440         writel(dmc->timing_data[idx],
 441                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);
 442         writel(dmc->timing_power[idx],
 443                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);
 444         writel(dmc->timing_power[idx],
 445                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);
 446
 447         return 0;
 448 }
 449
 450 /**
 451  * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC
 452  * @dmc:        device for which it is going to be set
 453  * @target_volt:        new voltage which is chosen to be final
 454  *
 455  * Function tries to align voltage to the safe level for 'normal' mode.
 456  * It checks the need of higher voltage and changes the value. The target
 457  * voltage might be lower that currently set and still the system will be
 458  * stable.
 459  */
 460 static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,
 461                                             unsigned long target_volt)
 462 {
 463         int ret = 0;
 464
 465         if (dmc->curr_volt <= target_volt)
 466                 return 0;
 467
 468         ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
 469                                     target_volt);
 470         if (!ret)
 471                 dmc->curr_volt = target_volt;
 472
 473         return ret;
 474 }
 475
 476 /**
 477  * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC
 478  * @dmc:        device for which it is going to be set
 479  * @target_volt:        new voltage which is chosen to be final
 480  *
 481  * Function tries to align voltage to the safe level for the 'bypass' mode.
 482  * It checks the need of higher voltage and changes the value.
 483  * The target voltage must not be less than currently needed, because
 484  * for current frequency the device might become unstable.
 485  */
 486 static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,
 487                                             unsigned long target_volt)
 488 {
 489         int ret = 0;
 490
 491         if (dmc->curr_volt >= target_volt)
 492                 return 0;
 493
 494         ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
 495                                     target_volt);
 496         if (!ret)
 497                 dmc->curr_volt = target_volt;
 498
 499         return ret;
 500 }
 501
 502 /**
 503  * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings
 504  * @dmc:        device for which it is going to be set
 505  * @target_rate:        new frequency which is chosen to be final
 506  *
 507  * Function changes the DRAM timings for the temporary 'bypass' mode.
 508  */
 509 static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,
 510                                                  unsigned long target_rate)
 511 {
 512         int idx = find_target_freq_idx(dmc, target_rate);
 513
 514         if (idx < 0)
 515                 return -EINVAL;
 516
 517         exynos5_set_bypass_dram_timings(dmc);
 518
 519         return 0;
 520 }
 521
 522 /**
 523  * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock
 524  * @dmc:        DMC device for which the switching is going to happen
 525  * @target_rate:        new frequency which is going to be set as a final
 526  * @target_volt:        new voltage which is going to be set as a final
 527  *
 528  * Function configures DMC and clocks for operating in temporary 'bypass' mode.
 529  * This mode is used only temporary but if required, changes voltage and timings
 530  * for DRAM chips. It switches the main clock to stable clock source for the
 531  * period of the main PLL reconfiguration.
 532  */
 533 static int
 534 exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,
 535                                            unsigned long target_rate,
 536                                            unsigned long target_volt)
 537 {
 538         int ret;
 539
 540         /*
 541          * Having higher voltage for a particular frequency does not harm
 542          * the chip. Use it for the temporary frequency change when one
 543          * voltage manipulation might be avoided.
 544          */
 545         ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);
 546         if (ret)
 547                 return ret;
 548
 549         /*
 550          * Longer delays for DRAM does not cause crash, the opposite does.
 551          */
 552         ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);
 553         if (ret)
 554                 return ret;
 555
 556         /*
 557          * Delays are long enough, so use them for the new coming clock.
 558          */
 559         ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);
 560
 561         return ret;
 562 }
 563
 564 /**
 565  * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC
 566  * using safe procedure
 567  * @dmc:        device for which the frequency is going to be changed
 568  * @target_rate:        requested new frequency
 569  * @target_volt:        requested voltage which corresponds to the new frequency
 570  *
 571  * The DMC frequency change procedure requires a few steps.
 572  * The main requirement is to change the clock source in the clk mux
 573  * for the time of main clock PLL locking. The assumption is that the
 574  * alternative clock source set as parent is stable.
 575  * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass'
 576  * clock. This requires alignment in DRAM timing parameters for the new
 577  * T-period. There is two bank sets for keeping DRAM
 578  * timings: set 0 and set 1. The set 0 is used when main clock source is
 579  * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between
 580  * the two bank sets is part of the process.
 581  * The voltage must also be aligned to the minimum required level. There is
 582  * this intermediate step with switching to 'bypass' parent clock source.
 583  * if the old voltage is lower, it requires an increase of the voltage level.
 584  * The complexity of the voltage manipulation is hidden in low level function.
 585  * In this function there is last alignment of the voltage level at the end.
 586  */
 587 static int
 588 exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,
 589                                  unsigned long target_rate,
 590                                  unsigned long target_volt)
 591 {
 592         int ret;
 593
 594         ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,
 595                                                          target_volt);
 596         if (ret)
 597                 return ret;
 598
 599         /*
 600          * Voltage is set at least to a level needed for this frequency,
 601          * so switching clock source is safe now.
 602          */
 603         clk_prepare_enable(dmc->fout_spll);
 604         clk_prepare_enable(dmc->mout_spll);
 605         clk_prepare_enable(dmc->mout_mx_mspll_ccore);
 606
 607         ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);
 608         if (ret)
 609                 goto disable_clocks;
 610
 611         /*
 612          * We are safe to increase the timings for current bypass frequency.
 613          * Thanks to this the settings will be ready for the upcoming clock
 614          * source change.
 615          */
 616         exynos5_dram_change_timings(dmc, target_rate);
 617
 618         clk_set_rate(dmc->fout_bpll, target_rate);
 619
 620         ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);
 621         if (ret)
 622                 goto disable_clocks;
 623
 624         ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);
 625         if (ret)
 626                 goto disable_clocks;
 627
 628         /*
 629          * Make sure if the voltage is not from 'bypass' settings and align to
 630          * the right level for power efficiency.
 631          */
 632         ret = exynos5_dmc_align_target_voltage(dmc, target_volt);
 633
 634 disable_clocks:
 635         clk_disable_unprepare(dmc->mout_mx_mspll_ccore);
 636         clk_disable_unprepare(dmc->mout_spll);
 637         clk_disable_unprepare(dmc->fout_spll);
 638
 639         return ret;
 640 }
 641
 642 /**
 643  * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP
 644  * table.
 645  * @dmc:        device for which the frequency is going to be changed
 646  * @freq:       requested frequency in KHz
 647  * @target_rate:        returned frequency which is the same or lower than
 648  *                      requested
 649  * @target_volt:        returned voltage which corresponds to the returned
 650  *                      frequency
 651  * @flags:      devfreq flags provided for this frequency change request
 652  *
 653  * Function gets requested frequency and checks OPP framework for needed
 654  * frequency and voltage. It populates the values 'target_rate' and
 655  * 'target_volt' or returns error value when OPP framework fails.
 656  */
 657 static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,
 658                                      unsigned long *freq,
 659                                      unsigned long *target_rate,
 660                                      unsigned long *target_volt, u32 flags)
 661 {
 662         struct dev_pm_opp *opp;
 663
 664         opp = devfreq_recommended_opp(dmc->dev, freq, flags);
 665         if (IS_ERR(opp))
 666                 return PTR_ERR(opp);
 667
 668         *target_rate = dev_pm_opp_get_freq(opp);
 669         *target_volt = dev_pm_opp_get_voltage(opp);
 670         dev_pm_opp_put(opp);
 671
 672         return 0;
 673 }
 674
 675 /**
 676  * exynos5_dmc_target() - Function responsible for changing frequency of DMC
 677  * @dev:        device for which the frequency is going to be changed
 678  * @freq:       requested frequency in KHz
 679  * @flags:      flags provided for this frequency change request
 680  *
 681  * An entry function provided to the devfreq framework which provides frequency
 682  * change of the DMC. The function gets the possible rate from OPP table based
 683  * on requested frequency. It calls the next function responsible for the
 684  * frequency and voltage change. In case of failure, does not set 'curr_rate'
 685  * and returns error value to the framework.
 686  */
 687 static int exynos5_dmc_target(struct device *dev, unsigned long *freq,
 688                               u32 flags)
 689 {
 690         struct exynos5_dmc *dmc = dev_get_drvdata(dev);
 691         unsigned long target_rate = 0;
 692         unsigned long target_volt = 0;
 693         int ret;
 694
 695         ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,
 696                                         flags);
 697
 698         if (ret)
 699                 return ret;
 700
 701         if (target_rate == dmc->curr_rate)
 702                 return 0;
 703
 704         mutex_lock(&dmc->lock);
 705
 706         ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);
 707
 708         if (ret) {
 709                 mutex_unlock(&dmc->lock);
 710                 return ret;
 711         }
 712
 713         dmc->curr_rate = target_rate;
 714
 715         mutex_unlock(&dmc->lock);
 716         return 0;
 717 }
 718
 719 /**
 720  * exynos5_counters_get() - Gets the performance counters values.
 721  * @dmc:        device for which the counters are going to be checked
 722  * @load_count: variable which is populated with counter value
 723  * @total_count:        variable which is used as 'wall clock' reference
 724  *
 725  * Function which provides performance counters values. It sums up counters for
 726  * two DMC channels. The 'total_count' is used as a reference and max value.
 727  * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%].
 728  */
 729 static int exynos5_counters_get(struct exynos5_dmc *dmc,
 730                                 unsigned long *load_count,
 731                                 unsigned long *total_count)
 732 {
 733         unsigned long total = 0;
 734         struct devfreq_event_data event;
 735         int ret, i;
 736
 737         *load_count = 0;
 738
 739         /* Take into account only read+write counters, but stop all */
 740         for (i = 0; i < dmc->num_counters; i++) {
 741                 if (!dmc->counter[i])
 742                         continue;
 743
 744                 ret = devfreq_event_get_event(dmc->counter[i], &event);
 745                 if (ret < 0)
 746                         return ret;
 747
 748                 *load_count += event.load_count;
 749
 750                 if (total < event.total_count)
 751                         total = event.total_count;
 752         }
 753
 754         *total_count = total;
 755
 756         return 0;
 757 }
 758
 759 /**
 760  * exynos5_dmc_start_perf_events() - Setup and start performance event counters
 761  * @dmc:        device for which the counters are going to be checked
 762  * @beg_value:  initial value for the counter
 763  *
 764  * Function which enables needed counters, interrupts and sets initial values
 765  * then starts the counters.
 766  */
 767 static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,
 768                                           u32 beg_value)
 769 {
 770         /* Enable interrupts for counter 2 */
 771         writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);
 772         writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);
 773
 774         /* Enable counter 2 and CCNT  */
 775         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);
 776         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);
 777
 778         /* Clear overflow flag for all counters */
 779         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
 780         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
 781
 782         /* Reset all counters */
 783         writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);
 784         writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);
 785
 786         /*
 787          * Set start value for the counters, the number of samples that
 788          * will be gathered is calculated as: 0xffffffff - beg_value
 789          */
 790         writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);
 791         writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);
 792
 793         /* Start all counters */
 794         writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);
 795         writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);
 796 }
 797
 798 /**
 799  * exynos5_dmc_perf_events_calc() - Calculate utilization
 800  * @dmc:        device for which the counters are going to be checked
 801  * @diff_ts:    time between last interrupt and current one
 802  *
 803  * Function which calculates needed utilization for the devfreq governor.
 804  * It prepares values for 'busy_time' and 'total_time' based on elapsed time
 805  * between interrupts, which approximates utilization.
 806  */
 807 static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)
 808 {
 809         /*
 810          * This is a simple algorithm for managing traffic on DMC.
 811          * When there is almost no load the counters overflow every 4s,
 812          * no mater the DMC frequency.
 813          * The high load might be approximated using linear function.
 814          * Knowing that, simple calculation can provide 'busy_time' and
 815          * 'total_time' to the devfreq governor which picks up target
 816          * frequency.
 817          * We want a fast ramp up and slow decay in frequency change function.
 818          */
 819         if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {
 820                 /*
 821                  * Set higher utilization for the simple_ondemand governor.
 822                  * The governor should increase the frequency of the DMC.
 823                  */
 824                 dmc->load = 70;
 825                 dmc->total = 100;
 826         } else {
 827                 /*
 828                  * Set low utilization for the simple_ondemand governor.
 829                  * The governor should decrease the frequency of the DMC.
 830                  */
 831                 dmc->load = 35;
 832                 dmc->total = 100;
 833         }
 834
 835         dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);
 836 }
 837
 838 /**
 839  * exynos5_dmc_perf_events_check() - Checks the status of the counters
 840  * @dmc:        device for which the counters are going to be checked
 841  *
 842  * Function which is called from threaded IRQ to check the counters state
 843  * and to call approximation for the needed utilization.
 844  */
 845 static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)
 846 {
 847         u32 val;
 848         u64 diff_ts, ts;
 849
 850         ts = ktime_get_ns();
 851
 852         /* Stop all counters */
 853         writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
 854         writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
 855
 856         /* Check the source in interrupt flag registers (which channel) */
 857         val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);
 858         if (val) {
 859                 diff_ts = ts - dmc->last_overflow_ts[0];
 860                 dmc->last_overflow_ts[0] = ts;
 861                 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n",  val);
 862         } else {
 863                 val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);
 864                 diff_ts = ts - dmc->last_overflow_ts[1];
 865                 dmc->last_overflow_ts[1] = ts;
 866                 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n",  val);
 867         }
 868
 869         exynos5_dmc_perf_events_calc(dmc, diff_ts);
 870
 871         exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
 872 }
 873
 874 /**
 875  * exynos5_dmc_enable_perf_events() - Enable performance events
 876  * @dmc:        device for which the counters are going to be checked
 877  *
 878  * Function which is setup needed environment and enables counters.
 879  */
 880 static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)
 881 {
 882         u64 ts;
 883
 884         /* Enable Performance Event Clock */
 885         writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);
 886         writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);
 887
 888         /* Select read transfers as performance event2 */
 889         writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);
 890         writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);
 891
 892         ts = ktime_get_ns();
 893         dmc->last_overflow_ts[0] = ts;
 894         dmc->last_overflow_ts[1] = ts;
 895
 896         /* Devfreq shouldn't be faster than initialization, play safe though. */
 897         dmc->load = 99;
 898         dmc->total = 100;
 899 }
 900
 901 /**
 902  * exynos5_dmc_disable_perf_events() - Disable performance events
 903  * @dmc:        device for which the counters are going to be checked
 904  *
 905  * Function which stops, disables performance event counters and interrupts.
 906  */
 907 static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)
 908 {
 909         /* Stop all counters */
 910         writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
 911         writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
 912
 913         /* Disable interrupts for counter 2 */
 914         writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);
 915         writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);
 916
 917         /* Disable counter 2 and CCNT  */
 918         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);
 919         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);
 920
 921         /* Clear overflow flag for all counters */
 922         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
 923         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
 924 }
 925
 926 /**
 927  * exynos5_dmc_get_status() - Read current DMC performance statistics.
 928  * @dev:        device for which the statistics are requested
 929  * @stat:       structure which has statistic fields
 930  *
 931  * Function reads the DMC performance counters and calculates 'busy_time'
 932  * and 'total_time'. To protect from overflow, the values are shifted right
 933  * by 10. After read out the counters are setup to count again.
 934  */
 935 static int exynos5_dmc_get_status(struct device *dev,
 936                                   struct devfreq_dev_status *stat)
 937 {
 938         struct exynos5_dmc *dmc = dev_get_drvdata(dev);
 939         unsigned long load, total;
 940         int ret;
 941
 942         if (dmc->in_irq_mode) {
 943                 mutex_lock(&dmc->lock);
 944                 stat->current_frequency = dmc->curr_rate;
 945                 mutex_unlock(&dmc->lock);
 946
 947                 stat->busy_time = dmc->load;
 948                 stat->total_time = dmc->total;
 949         } else {
 950                 ret = exynos5_counters_get(dmc, &load, &total);
 951                 if (ret < 0)
 952                         return -EINVAL;
 953
 954                 /* To protect from overflow, divide by 1024 */
 955                 stat->busy_time = load >> 10;
 956                 stat->total_time = total >> 10;
 957
 958                 ret = exynos5_counters_set_event(dmc);
 959                 if (ret < 0) {
 960                         dev_err(dev, "could not set event counter\n");
 961                         return ret;
 962                 }
 963         }
 964
 965         return 0;
 966 }
 967
 968 /**
 969  * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency
 970  * @dev:        device for which the framework checks operating frequency
 971  * @freq:       returned frequency value
 972  *
 973  * It returns the currently used frequency of the DMC. The real operating
 974  * frequency might be lower when the clock source value could not be divided
 975  * to the requested value.
 976  */
 977 static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)
 978 {
 979         struct exynos5_dmc *dmc = dev_get_drvdata(dev);
 980
 981         mutex_lock(&dmc->lock);
 982         *freq = dmc->curr_rate;
 983         mutex_unlock(&dmc->lock);
 984
 985         return 0;
 986 }
 987
 988 /*
 989  * exynos5_dmc_df_profile - Devfreq governor's profile structure
 990  *
 991  * It provides to the devfreq framework needed functions and polling period.
 992  */
 993 static struct devfreq_dev_profile exynos5_dmc_df_profile = {
 994         .timer = DEVFREQ_TIMER_DELAYED,
 995         .target = exynos5_dmc_target,
 996         .get_dev_status = exynos5_dmc_get_status,
 997         .get_cur_freq = exynos5_dmc_get_cur_freq,
 998 };
 999
1000 /**
1001  * exynos5_dmc_align_initial_frequency() - Align initial frequency value
1002  * @dmc:        device for which the frequency is going to be set
1003  * @bootloader_init_freq:       initial frequency set by the bootloader in KHz
1004  *
1005  * The initial bootloader frequency, which is present during boot, might be
1006  * different that supported frequency values in the driver. It is possible
1007  * due to different PLL settings or used PLL as a source.
1008  * This function provides the 'initial_freq' for the devfreq framework
1009  * statistics engine which supports only registered values. Thus, some alignment
1010  * must be made.
1011  */
1012 static unsigned long
1013 exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,
1014                             unsigned long bootloader_init_freq)
1015 {
1016         unsigned long aligned_freq;
1017         int idx;
1018
1019         idx = find_target_freq_idx(dmc, bootloader_init_freq);
1020         if (idx >= 0)
1021                 aligned_freq = dmc->opp[idx].freq_hz;
1022         else
1023                 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;
1024
1025         return aligned_freq;
1026 }
1027
1028 /**
1029  * create_timings_aligned() - Create register values and align with standard
1030  * @dmc:        device for which the frequency is going to be set
1031  * @reg_timing_row:     array to fill with values for timing row register
1032  * @reg_timing_data:    array to fill with values for timing data register
1033  * @reg_timing_power:   array to fill with values for timing power register
1034  * @clk_period_ps:      the period of the clock, known as tCK
1035  *
1036  * The function calculates timings and creates a register value ready for
1037  * a frequency transition. The register contains a few timings. They are
1038  * shifted by a known offset. The timing value is calculated based on memory
1039  * specyfication: minimal time required and minimal cycles required.
1040  */
1041 static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
1042                                   u32 *reg_timing_data, u32 *reg_timing_power,
1043                                   u32 clk_period_ps)
1044 {
1045         u32 val;
1046         const struct timing_reg *reg;
1047
1048         if (clk_period_ps == 0)
1049                 return -EINVAL;
1050
1051         *reg_timing_row = 0;
1052         *reg_timing_data = 0;
1053         *reg_timing_power = 0;
1054
1055         val = dmc->timings->tRFC / clk_period_ps;
1056         val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;
1057         val = max(val, dmc->min_tck->tRFC);
1058         reg = &timing_row_reg_fields[0];
1059         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1060
1061         val = dmc->timings->tRRD / clk_period_ps;
1062         val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;
1063         val = max(val, dmc->min_tck->tRRD);
1064         reg = &timing_row_reg_fields[1];
1065         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1066
1067         val = dmc->timings->tRPab / clk_period_ps;
1068         val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;
1069         val = max(val, dmc->min_tck->tRPab);
1070         reg = &timing_row_reg_fields[2];
1071         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1072
1073         val = dmc->timings->tRCD / clk_period_ps;
1074         val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;
1075         val = max(val, dmc->min_tck->tRCD);
1076         reg = &timing_row_reg_fields[3];
1077         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1078
1079         val = dmc->timings->tRC / clk_period_ps;
1080         val += dmc->timings->tRC % clk_period_ps ? 1 : 0;
1081         val = max(val, dmc->min_tck->tRC);
1082         reg = &timing_row_reg_fields[4];
1083         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1084
1085         val = dmc->timings->tRAS / clk_period_ps;
1086         val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;
1087         val = max(val, dmc->min_tck->tRAS);
1088         reg = &timing_row_reg_fields[5];
1089         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1090
1091         /* data related timings */
1092         val = dmc->timings->tWTR / clk_period_ps;
1093         val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;
1094         val = max(val, dmc->min_tck->tWTR);
1095         reg = &timing_data_reg_fields[0];
1096         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1097
1098         val = dmc->timings->tWR / clk_period_ps;
1099         val += dmc->timings->tWR % clk_period_ps ? 1 : 0;
1100         val = max(val, dmc->min_tck->tWR);
1101         reg = &timing_data_reg_fields[1];
1102         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1103
1104         val = dmc->timings->tRTP / clk_period_ps;
1105         val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;
1106         val = max(val, dmc->min_tck->tRTP);
1107         reg = &timing_data_reg_fields[2];
1108         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1109
1110         val = dmc->timings->tW2W_C2C / clk_period_ps;
1111         val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;
1112         val = max(val, dmc->min_tck->tW2W_C2C);
1113         reg = &timing_data_reg_fields[3];
1114         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1115
1116         val = dmc->timings->tR2R_C2C / clk_period_ps;
1117         val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;
1118         val = max(val, dmc->min_tck->tR2R_C2C);
1119         reg = &timing_data_reg_fields[4];
1120         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1121
1122         val = dmc->timings->tWL / clk_period_ps;
1123         val += dmc->timings->tWL % clk_period_ps ? 1 : 0;
1124         val = max(val, dmc->min_tck->tWL);
1125         reg = &timing_data_reg_fields[5];
1126         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1127
1128         val = dmc->timings->tDQSCK / clk_period_ps;
1129         val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;
1130         val = max(val, dmc->min_tck->tDQSCK);
1131         reg = &timing_data_reg_fields[6];
1132         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1133
1134         val = dmc->timings->tRL / clk_period_ps;
1135         val += dmc->timings->tRL % clk_period_ps ? 1 : 0;
1136         val = max(val, dmc->min_tck->tRL);
1137         reg = &timing_data_reg_fields[7];
1138         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1139
1140         /* power related timings */
1141         val = dmc->timings->tFAW / clk_period_ps;
1142         val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
1143         val = max(val, dmc->min_tck->tFAW);
1144         reg = &timing_power_reg_fields[0];
1145         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1146
1147         val = dmc->timings->tXSR / clk_period_ps;
1148         val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;
1149         val = max(val, dmc->min_tck->tXSR);
1150         reg = &timing_power_reg_fields[1];
1151         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1152
1153         val = dmc->timings->tXP / clk_period_ps;
1154         val += dmc->timings->tXP % clk_period_ps ? 1 : 0;
1155         val = max(val, dmc->min_tck->tXP);
1156         reg = &timing_power_reg_fields[2];
1157         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1158
1159         val = dmc->timings->tCKE / clk_period_ps;
1160         val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;
1161         val = max(val, dmc->min_tck->tCKE);
1162         reg = &timing_power_reg_fields[3];
1163         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1164
1165         val = dmc->timings->tMRD / clk_period_ps;
1166         val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;
1167         val = max(val, dmc->min_tck->tMRD);
1168         reg = &timing_power_reg_fields[4];
1169         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1170
1171         return 0;
1172 }
1173
1174 /**
1175  * of_get_dram_timings() - helper function for parsing DT settings for DRAM
1176  * @dmc:        device for which the frequency is going to be set
1177  *
1178  * The function parses DT entries with DRAM information.
1179  */
1180 static int of_get_dram_timings(struct exynos5_dmc *dmc)
1181 {
1182         int ret = 0;
1183         int idx;
1184         struct device_node *np_ddr;
1185         u32 freq_mhz, clk_period_ps;
1186
1187         np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0);
1188         if (!np_ddr) {
1189                 dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");
1190                 return -EINVAL;
1191         }
1192
1193         dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1194                                              sizeof(u32), GFP_KERNEL);
1195         if (!dmc->timing_row) {
1196                 ret = -ENOMEM;
1197                 goto put_node;
1198         }
1199
1200         dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1201                                               sizeof(u32), GFP_KERNEL);
1202         if (!dmc->timing_data) {
1203                 ret = -ENOMEM;
1204                 goto put_node;
1205         }
1206
1207         dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1208                                                sizeof(u32), GFP_KERNEL);
1209         if (!dmc->timing_power) {
1210                 ret = -ENOMEM;
1211                 goto put_node;
1212         }
1213
1214         dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev,
1215                                                  DDR_TYPE_LPDDR3,
1216                                                  &dmc->timings_arr_size);
1217         if (!dmc->timings) {
1218                 dev_warn(dmc->dev, "could not get timings from DT\n");
1219                 ret = -EINVAL;
1220                 goto put_node;
1221         }
1222
1223         dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev);
1224         if (!dmc->min_tck) {
1225                 dev_warn(dmc->dev, "could not get tck from DT\n");
1226                 ret = -EINVAL;
1227                 goto put_node;
1228         }
1229
1230         /* Sorted array of OPPs with frequency ascending */
1231         for (idx = 0; idx < dmc->opp_count; idx++) {
1232                 freq_mhz = dmc->opp[idx].freq_hz / 1000000;
1233                 clk_period_ps = 1000000 / freq_mhz;
1234
1235                 ret = create_timings_aligned(dmc, &dmc->timing_row[idx],
1236                                              &dmc->timing_data[idx],
1237                                              &dmc->timing_power[idx],
1238                                              clk_period_ps);
1239         }
1240
1241
1242         /* Take the highest frequency's timings as 'bypass' */
1243         dmc->bypass_timing_row = dmc->timing_row[idx - 1];
1244         dmc->bypass_timing_data = dmc->timing_data[idx - 1];
1245         dmc->bypass_timing_power = dmc->timing_power[idx - 1];
1246
1247 put_node:
1248         of_node_put(np_ddr);
1249         return ret;
1250 }
1251
1252 /**
1253  * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation.
1254  * @dmc:        DMC structure containing needed fields
1255  *
1256  * Get the needed clocks defined in DT device, enable and set the right parents.
1257  * Read current frequency and initialize the initial rate for governor.
1258  */
1259 static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
1260 {
1261         int ret;
1262         unsigned long target_volt = 0;
1263         unsigned long target_rate = 0;
1264         unsigned int tmp;
1265
1266         dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll");
1267         if (IS_ERR(dmc->fout_spll))
1268                 return PTR_ERR(dmc->fout_spll);
1269
1270         dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll");
1271         if (IS_ERR(dmc->fout_bpll))
1272                 return PTR_ERR(dmc->fout_bpll);
1273
1274         dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex");
1275         if (IS_ERR(dmc->mout_mclk_cdrex))
1276                 return PTR_ERR(dmc->mout_mclk_cdrex);
1277
1278         dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll");
1279         if (IS_ERR(dmc->mout_bpll))
1280                 return PTR_ERR(dmc->mout_bpll);
1281
1282         dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev,
1283                                                 "mout_mx_mspll_ccore");
1284         if (IS_ERR(dmc->mout_mx_mspll_ccore))
1285                 return PTR_ERR(dmc->mout_mx_mspll_ccore);
1286
1287         dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2");
1288         if (IS_ERR(dmc->mout_spll)) {
1289                 dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll");
1290                 if (IS_ERR(dmc->mout_spll))
1291                         return PTR_ERR(dmc->mout_spll);
1292         }
1293
1294         /*
1295          * Convert frequency to KHz values and set it for the governor.
1296          */
1297         dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);
1298         dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);
1299         exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;
1300
1301         ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,
1302                                         &target_volt, 0);
1303         if (ret)
1304                 return ret;
1305
1306         dmc->curr_volt = target_volt;
1307
1308         ret = clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);
1309         if (ret)
1310                 return ret;
1311
1312         clk_prepare_enable(dmc->fout_bpll);
1313         clk_prepare_enable(dmc->mout_bpll);
1314
1315         /*
1316          * Some bootloaders do not set clock routes correctly.
1317          * Stop one path in clocks to PHY.
1318          */
1319         regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);
1320         tmp &= ~(BIT(1) | BIT(0));
1321         regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);
1322
1323         return 0;
1324 }
1325
1326 /**
1327  * exynos5_performance_counters_init() - Initializes performance DMC's counters
1328  * @dmc:        DMC for which it does the setup
1329  *
1330  * Initialization of performance counters in DMC for estimating usage.
1331  * The counter's values are used for calculation of a memory bandwidth and based
1332  * on that the governor changes the frequency.
1333  * The counters are not used when the governor is GOVERNOR_USERSPACE.
1334  */
1335 static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)
1336 {
1337         int ret, i;
1338
1339         dmc->num_counters = devfreq_event_get_edev_count(dmc->dev,
1340                                                         "devfreq-events");
1341         if (dmc->num_counters < 0) {
1342                 dev_err(dmc->dev, "could not get devfreq-event counters\n");
1343                 return dmc->num_counters;
1344         }
1345
1346         dmc->counter = devm_kcalloc(dmc->dev, dmc->num_counters,
1347                                     sizeof(*dmc->counter), GFP_KERNEL);
1348         if (!dmc->counter)
1349                 return -ENOMEM;
1350
1351         for (i = 0; i < dmc->num_counters; i++) {
1352                 dmc->counter[i] =
1353                         devfreq_event_get_edev_by_phandle(dmc->dev,
1354                                                 "devfreq-events", i);
1355                 if (IS_ERR_OR_NULL(dmc->counter[i]))
1356                         return -EPROBE_DEFER;
1357         }
1358
1359         ret = exynos5_counters_enable_edev(dmc);
1360         if (ret < 0) {
1361                 dev_err(dmc->dev, "could not enable event counter\n");
1362                 return ret;
1363         }
1364
1365         ret = exynos5_counters_set_event(dmc);
1366         if (ret < 0) {
1367                 exynos5_counters_disable_edev(dmc);
1368                 dev_err(dmc->dev, "could not set event counter\n");
1369                 return ret;
1370         }
1371
1372         return 0;
1373 }
1374
1375 /**
1376  * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC
1377  * @dmc:        device which is used for changing this feature
1378  *
1379  * There is a need of pausing DREX DMC when divider or MUX in clock tree
1380  * changes its configuration. In such situation access to the memory is blocked
1381  * in DMC automatically. This feature is used when clock frequency change
1382  * request appears and touches clock tree.
1383  */
1384 static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)
1385 {
1386         unsigned int val;
1387         int ret;
1388
1389         ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);
1390         if (ret)
1391                 return ret;
1392
1393         val |= 1UL;
1394         regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);
1395
1396         return 0;
1397 }
1398
1399 static irqreturn_t dmc_irq_thread(int irq, void *priv)
1400 {
1401         int res;
1402         struct exynos5_dmc *dmc = priv;
1403
1404         mutex_lock(&dmc->df->lock);
1405         exynos5_dmc_perf_events_check(dmc);
1406         res = update_devfreq(dmc->df);
1407         mutex_unlock(&dmc->df->lock);
1408
1409         if (res)
1410                 dev_warn(dmc->dev, "devfreq failed with %d\n", res);
1411
1412         return IRQ_HANDLED;
1413 }
1414
1415 /**
1416  * exynos5_dmc_probe() - Probe function for the DMC driver
1417  * @pdev:       platform device for which the driver is going to be initialized
1418  *
1419  * Initialize basic components: clocks, regulators, performance counters, etc.
1420  * Read out product version and based on the information setup
1421  * internal structures for the controller (frequency and voltage) and for DRAM
1422  * memory parameters: timings for each operating frequency.
1423  * Register new devfreq device for controlling DVFS of the DMC.
1424  */
1425 static int exynos5_dmc_probe(struct platform_device *pdev)
1426 {
1427         int ret = 0;
1428         struct device *dev = &pdev->dev;
1429         struct device_node *np = dev->of_node;
1430         struct exynos5_dmc *dmc;
1431         int irq[2];
1432
1433         dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);
1434         if (!dmc)
1435                 return -ENOMEM;
1436
1437         mutex_init(&dmc->lock);
1438
1439         dmc->dev = dev;
1440         platform_set_drvdata(pdev, dmc);
1441
1442         dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);
1443         if (IS_ERR(dmc->base_drexi0))
1444                 return PTR_ERR(dmc->base_drexi0);
1445
1446         dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);
1447         if (IS_ERR(dmc->base_drexi1))
1448                 return PTR_ERR(dmc->base_drexi1);
1449
1450         dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,
1451                                                           "samsung,syscon-clk");
1452         if (IS_ERR(dmc->clk_regmap))
1453                 return PTR_ERR(dmc->clk_regmap);
1454
1455         ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);
1456         if (ret) {
1457                 dev_warn(dev, "couldn't initialize frequency settings\n");
1458                 return ret;
1459         }
1460
1461         dmc->vdd_mif = devm_regulator_get(dev, "vdd");
1462         if (IS_ERR(dmc->vdd_mif)) {
1463                 ret = PTR_ERR(dmc->vdd_mif);
1464                 return ret;
1465         }
1466
1467         ret = exynos5_dmc_init_clks(dmc);
1468         if (ret)
1469                 return ret;
1470
1471         ret = of_get_dram_timings(dmc);
1472         if (ret) {
1473                 dev_warn(dev, "couldn't initialize timings settings\n");
1474                 goto remove_clocks;
1475         }
1476
1477         ret = exynos5_dmc_set_pause_on_switching(dmc);
1478         if (ret) {
1479                 dev_warn(dev, "couldn't get access to PAUSE register\n");
1480                 goto remove_clocks;
1481         }
1482
1483         /* There is two modes in which the driver works: polling or IRQ */
1484         irq[0] = platform_get_irq_byname(pdev, "drex_0");
1485         irq[1] = platform_get_irq_byname(pdev, "drex_1");
1486         if (irq[0] > 0 && irq[1] > 0 && irqmode) {
1487                 ret = devm_request_threaded_irq(dev, irq[0], NULL,
1488                                                 dmc_irq_thread, IRQF_ONESHOT,
1489                                                 dev_name(dev), dmc);
1490                 if (ret) {
1491                         dev_err(dev, "couldn't grab IRQ\n");
1492                         goto remove_clocks;
1493                 }
1494
1495                 ret = devm_request_threaded_irq(dev, irq[1], NULL,
1496                                                 dmc_irq_thread, IRQF_ONESHOT,
1497                                                 dev_name(dev), dmc);
1498                 if (ret) {
1499                         dev_err(dev, "couldn't grab IRQ\n");
1500                         goto remove_clocks;
1501                 }
1502
1503                 /*
1504                  * Setup default thresholds for the devfreq governor.
1505                  * The values are chosen based on experiments.
1506                  */
1507                 dmc->gov_data.upthreshold = 55;
1508                 dmc->gov_data.downdifferential = 5;
1509
1510                 exynos5_dmc_enable_perf_events(dmc);
1511
1512                 dmc->in_irq_mode = 1;
1513         } else {
1514                 ret = exynos5_performance_counters_init(dmc);
1515                 if (ret) {
1516                         dev_warn(dev, "couldn't probe performance counters\n");
1517                         goto remove_clocks;
1518                 }
1519
1520                 /*
1521                  * Setup default thresholds for the devfreq governor.
1522                  * The values are chosen based on experiments.
1523                  */
1524                 dmc->gov_data.upthreshold = 10;
1525                 dmc->gov_data.downdifferential = 5;
1526
1527                 exynos5_dmc_df_profile.polling_ms = 100;
1528         }
1529
1530         dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,
1531                                           DEVFREQ_GOV_SIMPLE_ONDEMAND,
1532                                           &dmc->gov_data);
1533
1534         if (IS_ERR(dmc->df)) {
1535                 ret = PTR_ERR(dmc->df);
1536                 goto err_devfreq_add;
1537         }
1538
1539         if (dmc->in_irq_mode)
1540                 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
1541
1542         dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode);
1543
1544         return 0;
1545
1546 err_devfreq_add:
1547         if (dmc->in_irq_mode)
1548                 exynos5_dmc_disable_perf_events(dmc);
1549         else
1550                 exynos5_counters_disable_edev(dmc);
1551 remove_clocks:
1552         clk_disable_unprepare(dmc->mout_bpll);
1553         clk_disable_unprepare(dmc->fout_bpll);
1554
1555         return ret;
1556 }
1557
1558 /**
1559  * exynos5_dmc_remove() - Remove function for the platform device
1560  * @pdev:       platform device which is going to be removed
1561  *
1562  * The function relies on 'devm' framework function which automatically
1563  * clean the device's resources. It just calls explicitly disable function for
1564  * the performance counters.
1565  */
1566 static int exynos5_dmc_remove(struct platform_device *pdev)
1567 {
1568         struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);
1569
1570         if (dmc->in_irq_mode)
1571                 exynos5_dmc_disable_perf_events(dmc);
1572         else
1573                 exynos5_counters_disable_edev(dmc);
1574
1575         clk_disable_unprepare(dmc->mout_bpll);
1576         clk_disable_unprepare(dmc->fout_bpll);
1577
1578         dev_pm_opp_remove_table(dmc->dev);
1579
1580         return 0;
1581 }
1582
1583 static const struct of_device_id exynos5_dmc_of_match[] = {
1584         { .compatible = "samsung,exynos5422-dmc", },
1585         { },
1586 };
1587 MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);
1588
1589 static struct platform_driver exynos5_dmc_platdrv = {
1590         .probe  = exynos5_dmc_probe,
1591         .remove = exynos5_dmc_remove,
1592         .driver = {
1593                 .name   = "exynos5-dmc",
1594                 .of_match_table = exynos5_dmc_of_match,
1595         },
1596 };
1597 module_platform_driver(exynos5_dmc_platdrv);
1598 MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");
1599 MODULE_LICENSE("GPL v2");
1600 MODULE_AUTHOR("Lukasz Luba");