1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
6 #include <linux/clk/tegra.h>
7 #include <linux/genalloc.h>
8 #include <linux/mailbox_client.h>
9 #include <linux/module.h>
11 #include <linux/of_platform.h>
12 #include <linux/platform_device.h>
14 #include <linux/semaphore.h>
15 #include <linux/sched/clock.h>
17 #include <soc/tegra/bpmp.h>
18 #include <soc/tegra/bpmp-abi.h>
19 #include <soc/tegra/ivc.h>
21 #include "bpmp-private.h"
23 #define MSG_ACK BIT(0)
24 #define MSG_RING BIT(1)
27 static inline struct tegra_bpmp *
28 mbox_client_to_bpmp(struct mbox_client *client)
30 return container_of(client, struct tegra_bpmp, mbox.client);
33 static inline const struct tegra_bpmp_ops *
34 channel_to_ops(struct tegra_bpmp_channel *channel)
36 struct tegra_bpmp *bpmp = channel->bpmp;
38 return bpmp->soc->ops;
41 struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
43 struct platform_device *pdev;
44 struct tegra_bpmp *bpmp;
45 struct device_node *np;
47 np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
49 return ERR_PTR(-ENOENT);
51 pdev = of_find_device_by_node(np);
53 bpmp = ERR_PTR(-ENODEV);
57 bpmp = platform_get_drvdata(pdev);
59 bpmp = ERR_PTR(-EPROBE_DEFER);
60 put_device(&pdev->dev);
68 EXPORT_SYMBOL_GPL(tegra_bpmp_get);
70 void tegra_bpmp_put(struct tegra_bpmp *bpmp)
73 put_device(bpmp->dev);
75 EXPORT_SYMBOL_GPL(tegra_bpmp_put);
78 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
80 struct tegra_bpmp *bpmp = channel->bpmp;
84 count = bpmp->soc->channels.thread.count;
86 index = channel - channel->bpmp->threaded_channels;
87 if (index < 0 || index >= count)
93 static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
95 return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
96 (msg->rx.size <= MSG_DATA_MIN_SZ) &&
97 (msg->tx.size == 0 || msg->tx.data) &&
98 (msg->rx.size == 0 || msg->rx.data);
101 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
103 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
105 return ops->is_response_ready(channel);
108 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
110 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
112 return ops->is_request_ready(channel);
115 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
117 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
120 end = ktime_add_us(ktime_get(), timeout);
123 if (tegra_bpmp_is_response_ready(channel))
125 } while (ktime_before(ktime_get(), end));
130 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
132 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
134 return ops->ack_response(channel);
137 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
139 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
141 return ops->ack_request(channel);
145 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
147 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
149 return ops->is_request_channel_free(channel);
153 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
155 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
157 return ops->is_response_channel_free(channel);
161 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
163 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
166 start = ns_to_ktime(local_clock());
169 if (tegra_bpmp_is_request_channel_free(channel))
172 now = ns_to_ktime(local_clock());
173 } while (ktime_us_delta(now, start) < timeout);
178 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
180 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
182 return ops->post_request(channel);
185 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
187 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
189 return ops->post_response(channel);
192 static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
194 return bpmp->soc->ops->ring_doorbell(bpmp);
197 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
198 void *data, size_t size, int *ret)
202 if (data && size > 0)
203 tegra_bpmp_mb_read(data, &channel->ib, size);
205 err = tegra_bpmp_ack_response(channel);
209 *ret = tegra_bpmp_mb_read_field(&channel->ib, code);
214 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
215 void *data, size_t size, int *ret)
217 struct tegra_bpmp *bpmp = channel->bpmp;
222 index = tegra_bpmp_channel_get_thread_index(channel);
228 spin_lock_irqsave(&bpmp->lock, flags);
229 err = __tegra_bpmp_channel_read(channel, data, size, ret);
230 clear_bit(index, bpmp->threaded.allocated);
231 spin_unlock_irqrestore(&bpmp->lock, flags);
234 up(&bpmp->threaded.lock);
239 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
240 unsigned int mrq, unsigned long flags,
241 const void *data, size_t size)
243 tegra_bpmp_mb_write_field(&channel->ob, code, mrq);
244 tegra_bpmp_mb_write_field(&channel->ob, flags, flags);
246 if (data && size > 0)
247 tegra_bpmp_mb_write(&channel->ob, data, size);
249 return tegra_bpmp_post_request(channel);
252 static struct tegra_bpmp_channel *
253 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
254 const void *data, size_t size)
256 unsigned long timeout = bpmp->soc->channels.thread.timeout;
257 unsigned int count = bpmp->soc->channels.thread.count;
258 struct tegra_bpmp_channel *channel;
263 err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
267 spin_lock_irqsave(&bpmp->lock, flags);
269 index = find_first_zero_bit(bpmp->threaded.allocated, count);
270 if (index == count) {
275 channel = &bpmp->threaded_channels[index];
277 if (!tegra_bpmp_is_request_channel_free(channel)) {
282 set_bit(index, bpmp->threaded.allocated);
284 err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
287 goto clear_allocated;
289 set_bit(index, bpmp->threaded.busy);
291 spin_unlock_irqrestore(&bpmp->lock, flags);
295 clear_bit(index, bpmp->threaded.allocated);
297 spin_unlock_irqrestore(&bpmp->lock, flags);
298 up(&bpmp->threaded.lock);
303 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
304 unsigned int mrq, unsigned long flags,
305 const void *data, size_t size)
309 err = tegra_bpmp_wait_request_channel_free(channel);
313 return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
316 static int __maybe_unused tegra_bpmp_resume(struct device *dev);
318 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
319 struct tegra_bpmp_message *msg)
321 struct tegra_bpmp_channel *channel;
324 if (WARN_ON(!irqs_disabled()))
327 if (!tegra_bpmp_message_valid(msg))
330 if (bpmp->suspended) {
331 /* Reset BPMP IPC channels during resume based on flags passed */
332 if (msg->flags & TEGRA_BPMP_MESSAGE_RESET)
333 tegra_bpmp_resume(bpmp->dev);
338 channel = bpmp->tx_channel;
340 spin_lock(&bpmp->atomic_tx_lock);
342 err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
343 msg->tx.data, msg->tx.size);
345 spin_unlock(&bpmp->atomic_tx_lock);
349 spin_unlock(&bpmp->atomic_tx_lock);
351 err = tegra_bpmp_ring_doorbell(bpmp);
355 err = tegra_bpmp_wait_response(channel);
359 return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
362 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
364 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
365 struct tegra_bpmp_message *msg)
367 struct tegra_bpmp_channel *channel;
368 unsigned long timeout;
371 if (WARN_ON(irqs_disabled()))
374 if (!tegra_bpmp_message_valid(msg))
377 if (bpmp->suspended) {
378 /* Reset BPMP IPC channels during resume based on flags passed */
379 if (msg->flags & TEGRA_BPMP_MESSAGE_RESET)
380 tegra_bpmp_resume(bpmp->dev);
385 channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
388 return PTR_ERR(channel);
390 err = tegra_bpmp_ring_doorbell(bpmp);
394 timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
396 err = wait_for_completion_timeout(&channel->completion, timeout);
400 return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
403 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
405 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
408 struct tegra_bpmp_mrq *entry;
410 list_for_each_entry(entry, &bpmp->mrqs, list)
411 if (entry->mrq == mrq)
417 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
418 const void *data, size_t size)
420 unsigned long flags = tegra_bpmp_mb_read_field(&channel->ib, flags);
421 struct tegra_bpmp *bpmp = channel->bpmp;
424 if (WARN_ON(size > MSG_DATA_MIN_SZ))
427 err = tegra_bpmp_ack_request(channel);
428 if (WARN_ON(err < 0))
431 if ((flags & MSG_ACK) == 0)
434 if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
437 tegra_bpmp_mb_write_field(&channel->ob, code, code);
439 if (data && size > 0)
440 tegra_bpmp_mb_write(&channel->ob, data, size);
442 err = tegra_bpmp_post_response(channel);
443 if (WARN_ON(err < 0))
446 if (flags & MSG_RING) {
447 err = tegra_bpmp_ring_doorbell(bpmp);
448 if (WARN_ON(err < 0))
452 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
454 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
456 struct tegra_bpmp_channel *channel)
458 struct tegra_bpmp_mrq *entry;
461 spin_lock(&bpmp->lock);
463 entry = tegra_bpmp_find_mrq(bpmp, mrq);
465 spin_unlock(&bpmp->lock);
466 tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
470 entry->handler(mrq, channel, entry->data);
472 spin_unlock(&bpmp->lock);
475 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
476 tegra_bpmp_mrq_handler_t handler, void *data)
478 struct tegra_bpmp_mrq *entry;
484 entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
488 spin_lock_irqsave(&bpmp->lock, flags);
491 entry->handler = handler;
493 list_add(&entry->list, &bpmp->mrqs);
495 spin_unlock_irqrestore(&bpmp->lock, flags);
499 EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
501 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
503 struct tegra_bpmp_mrq *entry;
506 spin_lock_irqsave(&bpmp->lock, flags);
508 entry = tegra_bpmp_find_mrq(bpmp, mrq);
512 list_del(&entry->list);
513 devm_kfree(bpmp->dev, entry);
516 spin_unlock_irqrestore(&bpmp->lock, flags);
518 EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
520 bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
522 struct mrq_query_abi_request req = { .mrq = mrq };
523 struct mrq_query_abi_response resp;
524 struct tegra_bpmp_message msg = {
525 .mrq = MRQ_QUERY_ABI,
532 .size = sizeof(resp),
537 err = tegra_bpmp_transfer(bpmp, &msg);
538 if (err || msg.rx.ret)
541 return resp.status == 0;
543 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
545 static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
546 struct tegra_bpmp_channel *channel,
549 struct mrq_ping_request request;
550 struct mrq_ping_response response;
552 tegra_bpmp_mb_read(&request, &channel->ib, sizeof(request));
554 memset(&response, 0, sizeof(response));
555 response.reply = request.challenge << 1;
557 tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
560 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
562 struct mrq_ping_response response;
563 struct mrq_ping_request request;
564 struct tegra_bpmp_message msg;
569 memset(&request, 0, sizeof(request));
570 request.challenge = 1;
572 memset(&response, 0, sizeof(response));
574 memset(&msg, 0, sizeof(msg));
576 msg.tx.data = &request;
577 msg.tx.size = sizeof(request);
578 msg.rx.data = &response;
579 msg.rx.size = sizeof(response);
581 local_irq_save(flags);
583 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
585 local_irq_restore(flags);
589 "ping ok: challenge: %u, response: %u, time: %lld\n",
590 request.challenge, response.reply,
591 ktime_to_us(ktime_sub(end, start)));
596 /* deprecated version of tag query */
597 static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
600 struct mrq_query_tag_request request;
601 struct tegra_bpmp_message msg;
610 virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
611 GFP_KERNEL | GFP_DMA32);
615 memset(&request, 0, sizeof(request));
618 memset(&msg, 0, sizeof(msg));
619 msg.mrq = MRQ_QUERY_TAG;
620 msg.tx.data = &request;
621 msg.tx.size = sizeof(request);
623 local_irq_save(flags);
624 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
625 local_irq_restore(flags);
628 memcpy(tag, virt, TAG_SZ);
630 dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
635 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
638 if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
639 struct mrq_query_fw_tag_response resp;
640 struct tegra_bpmp_message msg = {
641 .mrq = MRQ_QUERY_FW_TAG,
644 .size = sizeof(resp),
649 if (size != sizeof(resp.tag))
652 err = tegra_bpmp_transfer(bpmp, &msg);
659 memcpy(tag, resp.tag, sizeof(resp.tag));
663 return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
666 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
668 unsigned long flags = tegra_bpmp_mb_read_field(&channel->ob, flags);
670 if ((flags & MSG_RING) == 0)
673 complete(&channel->completion);
676 void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
678 struct tegra_bpmp_channel *channel;
679 unsigned int i, count;
682 channel = bpmp->rx_channel;
683 count = bpmp->soc->channels.thread.count;
684 busy = bpmp->threaded.busy;
686 if (tegra_bpmp_is_request_ready(channel)) {
687 unsigned int mrq = tegra_bpmp_mb_read_field(&channel->ib, code);
689 tegra_bpmp_handle_mrq(bpmp, mrq, channel);
692 spin_lock(&bpmp->lock);
694 for_each_set_bit(i, busy, count) {
695 struct tegra_bpmp_channel *channel;
697 channel = &bpmp->threaded_channels[i];
699 if (tegra_bpmp_is_response_ready(channel)) {
700 tegra_bpmp_channel_signal(channel);
705 spin_unlock(&bpmp->lock);
708 static int tegra_bpmp_probe(struct platform_device *pdev)
710 struct tegra_bpmp *bpmp;
715 bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
719 bpmp->soc = of_device_get_match_data(&pdev->dev);
720 bpmp->dev = &pdev->dev;
722 INIT_LIST_HEAD(&bpmp->mrqs);
723 spin_lock_init(&bpmp->lock);
725 bpmp->threaded.count = bpmp->soc->channels.thread.count;
726 sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
728 size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
730 bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
731 if (!bpmp->threaded.allocated)
734 bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
735 if (!bpmp->threaded.busy)
738 spin_lock_init(&bpmp->atomic_tx_lock);
739 bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
741 if (!bpmp->tx_channel)
744 bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
746 if (!bpmp->rx_channel)
749 bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
750 sizeof(*bpmp->threaded_channels),
752 if (!bpmp->threaded_channels)
755 platform_set_drvdata(pdev, bpmp);
757 err = bpmp->soc->ops->init(bpmp);
761 err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
762 tegra_bpmp_mrq_handle_ping, bpmp);
766 err = tegra_bpmp_ping(bpmp);
768 dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
772 err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
774 dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
778 dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
780 err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
784 if (of_property_present(pdev->dev.of_node, "#clock-cells")) {
785 err = tegra_bpmp_init_clocks(bpmp);
790 if (of_property_present(pdev->dev.of_node, "#reset-cells")) {
791 err = tegra_bpmp_init_resets(bpmp);
796 if (of_property_present(pdev->dev.of_node, "#power-domain-cells")) {
797 err = tegra_bpmp_init_powergates(bpmp);
802 err = tegra_bpmp_init_debugfs(bpmp);
804 dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
809 tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
811 if (bpmp->soc->ops->deinit)
812 bpmp->soc->ops->deinit(bpmp);
817 static int __maybe_unused tegra_bpmp_suspend(struct device *dev)
819 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
821 bpmp->suspended = true;
826 static int __maybe_unused tegra_bpmp_resume(struct device *dev)
828 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
830 bpmp->suspended = false;
832 if (bpmp->soc->ops->resume)
833 return bpmp->soc->ops->resume(bpmp);
838 static const struct dev_pm_ops tegra_bpmp_pm_ops = {
839 .suspend_noirq = tegra_bpmp_suspend,
840 .resume_noirq = tegra_bpmp_resume,
843 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
844 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
845 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
846 static const struct tegra_bpmp_soc tegra186_soc = {
850 .timeout = 60 * USEC_PER_SEC,
855 .timeout = 600 * USEC_PER_SEC,
862 .ops = &tegra186_bpmp_ops,
867 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
868 static const struct tegra_bpmp_soc tegra210_soc = {
873 .timeout = 60 * USEC_PER_SEC,
878 .timeout = 600 * USEC_PER_SEC,
886 .ops = &tegra210_bpmp_ops,
890 static const struct of_device_id tegra_bpmp_match[] = {
891 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
892 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
893 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
894 { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
896 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
897 { .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
902 static struct platform_driver tegra_bpmp_driver = {
904 .name = "tegra-bpmp",
905 .of_match_table = tegra_bpmp_match,
906 .pm = &tegra_bpmp_pm_ops,
907 .suppress_bind_attrs = true,
909 .probe = tegra_bpmp_probe,
911 builtin_platform_driver(tegra_bpmp_driver);