1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
6 #include <linux/atomic.h>
8 #include <linux/interrupt.h>
9 #include <linux/jiffies.h>
10 #include <linux/kernel.h>
11 #include <linux/list.h>
12 #include <linux/lockdep.h>
13 #include <linux/module.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/types.h>
19 #include <linux/wait.h>
21 #include <soc/qcom/rpmh.h>
23 #include "rpmh-internal.h"
25 #define RPMH_TIMEOUT_MS msecs_to_jiffies(10000)
27 #define DEFINE_RPMH_MSG_ONSTACK(device, s, q, name) \
28 struct rpmh_request name = { \
33 .wait_for_compl = true, \
38 .needs_free = false, \
41 #define ctrlr_to_drv(ctrlr) container_of(ctrlr, struct rsc_drv, client)
44 * struct cache_req: the request object for caching
46 * @addr: the address of the resource
47 * @sleep_val: the sleep vote
48 * @wake_val: the wake vote
49 * @list: linked list obj
55 struct list_head list;
59 * struct batch_cache_req - An entry in our batch catch
61 * @list: linked list obj
62 * @count: number of messages
63 * @rpm_msgs: the messages
66 struct batch_cache_req {
67 struct list_head list;
69 struct rpmh_request rpm_msgs[];
72 static struct rpmh_ctrlr *get_rpmh_ctrlr(const struct device *dev)
74 struct rsc_drv *drv = dev_get_drvdata(dev->parent);
79 void rpmh_tx_done(const struct tcs_request *msg, int r)
81 struct rpmh_request *rpm_msg = container_of(msg, struct rpmh_request,
83 struct completion *compl = rpm_msg->completion;
84 bool free = rpm_msg->needs_free;
89 dev_err(rpm_msg->dev, "RPMH TX fail in msg addr=%#x, err=%d\n",
90 rpm_msg->msg.cmds[0].addr, r);
95 /* Signal the blocking thread we are done */
103 static struct cache_req *__find_req(struct rpmh_ctrlr *ctrlr, u32 addr)
105 struct cache_req *p, *req = NULL;
107 list_for_each_entry(p, &ctrlr->cache, list) {
108 if (p->addr == addr) {
117 static struct cache_req *cache_rpm_request(struct rpmh_ctrlr *ctrlr,
118 enum rpmh_state state,
121 struct cache_req *req;
123 u32 old_sleep_val, old_wake_val;
125 spin_lock_irqsave(&ctrlr->cache_lock, flags);
126 req = __find_req(ctrlr, cmd->addr);
130 req = kzalloc(sizeof(*req), GFP_ATOMIC);
132 req = ERR_PTR(-ENOMEM);
136 req->addr = cmd->addr;
137 req->sleep_val = req->wake_val = UINT_MAX;
138 list_add_tail(&req->list, &ctrlr->cache);
141 old_sleep_val = req->sleep_val;
142 old_wake_val = req->wake_val;
145 case RPMH_ACTIVE_ONLY_STATE:
146 case RPMH_WAKE_ONLY_STATE:
147 req->wake_val = cmd->data;
149 case RPMH_SLEEP_STATE:
150 req->sleep_val = cmd->data;
154 ctrlr->dirty |= (req->sleep_val != old_sleep_val ||
155 req->wake_val != old_wake_val) &&
156 req->sleep_val != UINT_MAX &&
157 req->wake_val != UINT_MAX;
160 spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
166 * __rpmh_write: Cache and send the RPMH request
168 * @dev: The device making the request
169 * @state: Active/Sleep request type
170 * @rpm_msg: The data that needs to be sent (cmds).
172 * Cache the RPMH request and send if the state is ACTIVE_ONLY.
173 * SLEEP/WAKE_ONLY requests are not sent to the controller at
174 * this time. Use rpmh_flush() to send them to the controller.
176 static int __rpmh_write(const struct device *dev, enum rpmh_state state,
177 struct rpmh_request *rpm_msg)
179 struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
181 struct cache_req *req;
184 rpm_msg->msg.state = state;
186 /* Cache the request in our store and link the payload */
187 for (i = 0; i < rpm_msg->msg.num_cmds; i++) {
188 req = cache_rpm_request(ctrlr, state, &rpm_msg->msg.cmds[i]);
193 rpm_msg->msg.state = state;
195 if (state == RPMH_ACTIVE_ONLY_STATE) {
196 WARN_ON(irqs_disabled());
197 ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg);
199 /* Clean up our call by spoofing tx_done */
201 rpmh_tx_done(&rpm_msg->msg, ret);
207 static int __fill_rpmh_msg(struct rpmh_request *req, enum rpmh_state state,
208 const struct tcs_cmd *cmd, u32 n)
210 if (!cmd || !n || n > MAX_RPMH_PAYLOAD)
213 memcpy(req->cmd, cmd, n * sizeof(*cmd));
215 req->msg.state = state;
216 req->msg.cmds = req->cmd;
217 req->msg.num_cmds = n;
223 * rpmh_write_async: Write a set of RPMH commands
225 * @dev: The device making the request
226 * @state: Active/sleep set
227 * @cmd: The payload data
228 * @n: The number of elements in payload
230 * Write a set of RPMH commands, the order of commands is maintained
231 * and will be sent as a single shot.
233 int rpmh_write_async(const struct device *dev, enum rpmh_state state,
234 const struct tcs_cmd *cmd, u32 n)
236 struct rpmh_request *rpm_msg;
239 rpm_msg = kzalloc(sizeof(*rpm_msg), GFP_ATOMIC);
242 rpm_msg->needs_free = true;
244 ret = __fill_rpmh_msg(rpm_msg, state, cmd, n);
250 return __rpmh_write(dev, state, rpm_msg);
252 EXPORT_SYMBOL(rpmh_write_async);
255 * rpmh_write: Write a set of RPMH commands and block until response
257 * @rc: The RPMH handle got from rpmh_get_client
258 * @state: Active/sleep set
259 * @cmd: The payload data
260 * @n: The number of elements in @cmd
262 * May sleep. Do not call from atomic contexts.
264 int rpmh_write(const struct device *dev, enum rpmh_state state,
265 const struct tcs_cmd *cmd, u32 n)
267 DECLARE_COMPLETION_ONSTACK(compl);
268 DEFINE_RPMH_MSG_ONSTACK(dev, state, &compl, rpm_msg);
271 if (!cmd || !n || n > MAX_RPMH_PAYLOAD)
274 memcpy(rpm_msg.cmd, cmd, n * sizeof(*cmd));
275 rpm_msg.msg.num_cmds = n;
277 ret = __rpmh_write(dev, state, &rpm_msg);
281 ret = wait_for_completion_timeout(&compl, RPMH_TIMEOUT_MS);
283 return (ret > 0) ? 0 : -ETIMEDOUT;
285 EXPORT_SYMBOL(rpmh_write);
287 static void cache_batch(struct rpmh_ctrlr *ctrlr, struct batch_cache_req *req)
291 spin_lock_irqsave(&ctrlr->cache_lock, flags);
292 list_add_tail(&req->list, &ctrlr->batch_cache);
294 spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
297 static int flush_batch(struct rpmh_ctrlr *ctrlr)
299 struct batch_cache_req *req;
300 const struct rpmh_request *rpm_msg;
304 /* Send Sleep/Wake requests to the controller, expect no response */
305 list_for_each_entry(req, &ctrlr->batch_cache, list) {
306 for (i = 0; i < req->count; i++) {
307 rpm_msg = req->rpm_msgs + i;
308 ret = rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr),
319 * rpmh_write_batch: Write multiple sets of RPMH commands and wait for the
322 * @dev: the device making the request
323 * @state: Active/sleep set
324 * @cmd: The payload data
325 * @n: The array of count of elements in each batch, 0 terminated.
327 * Write a request to the RSC controller without caching. If the request
328 * state is ACTIVE, then the requests are treated as completion request
329 * and sent to the controller immediately. The function waits until all the
330 * commands are complete. If the request was to SLEEP or WAKE_ONLY, then the
331 * request is sent as fire-n-forget and no ack is expected.
333 * May sleep. Do not call from atomic contexts for ACTIVE_ONLY requests.
335 int rpmh_write_batch(const struct device *dev, enum rpmh_state state,
336 const struct tcs_cmd *cmd, u32 *n)
338 struct batch_cache_req *req;
339 struct rpmh_request *rpm_msgs;
340 struct completion *compls;
341 struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
342 unsigned long time_left;
355 ptr = kzalloc(sizeof(*req) +
356 count * (sizeof(req->rpm_msgs[0]) + sizeof(*compls)),
362 compls = ptr + sizeof(*req) + count * sizeof(*rpm_msgs);
365 rpm_msgs = req->rpm_msgs;
367 for (i = 0; i < count; i++) {
368 __fill_rpmh_msg(rpm_msgs + i, state, cmd, n[i]);
372 if (state != RPMH_ACTIVE_ONLY_STATE) {
373 cache_batch(ctrlr, req);
377 for (i = 0; i < count; i++) {
378 struct completion *compl = &compls[i];
380 init_completion(compl);
381 rpm_msgs[i].completion = compl;
382 ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg);
384 pr_err("Error(%d) sending RPMH message addr=%#x\n",
385 ret, rpm_msgs[i].msg.cmds[0].addr);
390 time_left = RPMH_TIMEOUT_MS;
392 time_left = wait_for_completion_timeout(&compls[i], time_left);
395 * Better hope they never finish because they'll signal
396 * the completion that we're going to free once
397 * we've returned from this function.
410 EXPORT_SYMBOL(rpmh_write_batch);
412 static int is_req_valid(struct cache_req *req)
414 return (req->sleep_val != UINT_MAX &&
415 req->wake_val != UINT_MAX &&
416 req->sleep_val != req->wake_val);
419 static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
422 DEFINE_RPMH_MSG_ONSTACK(NULL, state, NULL, rpm_msg);
424 /* Wake sets are always complete and sleep sets are not */
425 rpm_msg.msg.wait_for_compl = (state == RPMH_WAKE_ONLY_STATE);
426 rpm_msg.cmd[0].addr = addr;
427 rpm_msg.cmd[0].data = data;
428 rpm_msg.msg.num_cmds = 1;
430 return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg);
434 * rpmh_flush() - Flushes the buffered sleep and wake sets to TCSes
436 * @ctrlr: Controller making request to flush cached data
440 * * Error code - Otherwise
442 int rpmh_flush(struct rpmh_ctrlr *ctrlr)
447 lockdep_assert_irqs_disabled();
450 * Currently rpmh_flush() is only called when we think we're running
451 * on the last processor. If the lock is busy it means another
452 * processor is up and it's better to abort than spin.
454 if (!spin_trylock(&ctrlr->cache_lock))
458 pr_debug("Skipping flush, TCS has latest data.\n");
462 /* Invalidate the TCSes first to avoid stale data */
463 rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));
465 /* First flush the cached batch requests */
466 ret = flush_batch(ctrlr);
470 list_for_each_entry(p, &ctrlr->cache, list) {
471 if (!is_req_valid(p)) {
472 pr_debug("%s: skipping RPMH req: a:%#x s:%#x w:%#x",
473 __func__, p->addr, p->sleep_val, p->wake_val);
476 ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
480 ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
486 ctrlr->dirty = false;
489 spin_unlock(&ctrlr->cache_lock);
494 * rpmh_invalidate: Invalidate sleep and wake sets in batch_cache
496 * @dev: The device making the request
498 * Invalidate the sleep and wake values in batch_cache.
500 void rpmh_invalidate(const struct device *dev)
502 struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
503 struct batch_cache_req *req, *tmp;
506 spin_lock_irqsave(&ctrlr->cache_lock, flags);
507 list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list)
509 INIT_LIST_HEAD(&ctrlr->batch_cache);
511 spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
513 EXPORT_SYMBOL(rpmh_invalidate);