2 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #include <linux/clk/tegra.h>
15 #include <linux/genalloc.h>
16 #include <linux/mailbox_client.h>
18 #include <linux/of_address.h>
19 #include <linux/of_device.h>
20 #include <linux/platform_device.h>
21 #include <linux/semaphore.h>
22 #include <linux/sched/clock.h>
24 #include <soc/tegra/bpmp.h>
25 #include <soc/tegra/bpmp-abi.h>
26 #include <soc/tegra/ivc.h>
28 #define MSG_ACK BIT(0)
29 #define MSG_RING BIT(1)
31 static inline struct tegra_bpmp *
32 mbox_client_to_bpmp(struct mbox_client *client)
34 return container_of(client, struct tegra_bpmp, mbox.client);
37 struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
39 struct platform_device *pdev;
40 struct tegra_bpmp *bpmp;
41 struct device_node *np;
43 np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
45 return ERR_PTR(-ENOENT);
47 pdev = of_find_device_by_node(np);
49 bpmp = ERR_PTR(-ENODEV);
53 bpmp = platform_get_drvdata(pdev);
55 bpmp = ERR_PTR(-EPROBE_DEFER);
56 put_device(&pdev->dev);
64 EXPORT_SYMBOL_GPL(tegra_bpmp_get);
66 void tegra_bpmp_put(struct tegra_bpmp *bpmp)
69 put_device(bpmp->dev);
71 EXPORT_SYMBOL_GPL(tegra_bpmp_put);
73 static int tegra_bpmp_channel_get_index(struct tegra_bpmp_channel *channel)
75 return channel - channel->bpmp->channels;
79 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
81 struct tegra_bpmp *bpmp = channel->bpmp;
82 unsigned int offset, count;
85 offset = bpmp->soc->channels.thread.offset;
86 count = bpmp->soc->channels.thread.count;
88 index = tegra_bpmp_channel_get_index(channel);
92 if (index < offset || index >= offset + count)
95 return index - offset;
98 static struct tegra_bpmp_channel *
99 tegra_bpmp_channel_get_thread(struct tegra_bpmp *bpmp, unsigned int index)
101 unsigned int offset = bpmp->soc->channels.thread.offset;
102 unsigned int count = bpmp->soc->channels.thread.count;
107 return &bpmp->channels[offset + index];
110 static struct tegra_bpmp_channel *
111 tegra_bpmp_channel_get_tx(struct tegra_bpmp *bpmp)
113 unsigned int offset = bpmp->soc->channels.cpu_tx.offset;
115 return &bpmp->channels[offset + smp_processor_id()];
118 static struct tegra_bpmp_channel *
119 tegra_bpmp_channel_get_rx(struct tegra_bpmp *bpmp)
121 unsigned int offset = bpmp->soc->channels.cpu_rx.offset;
123 return &bpmp->channels[offset];
126 static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
128 return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
129 (msg->rx.size <= MSG_DATA_MIN_SZ) &&
130 (msg->tx.size == 0 || msg->tx.data) &&
131 (msg->rx.size == 0 || msg->rx.data);
134 static bool tegra_bpmp_master_acked(struct tegra_bpmp_channel *channel)
138 frame = tegra_ivc_read_get_next_frame(channel->ivc);
149 static int tegra_bpmp_wait_ack(struct tegra_bpmp_channel *channel)
151 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
154 end = ktime_add_us(ktime_get(), timeout);
157 if (tegra_bpmp_master_acked(channel))
159 } while (ktime_before(ktime_get(), end));
164 static bool tegra_bpmp_master_free(struct tegra_bpmp_channel *channel)
168 frame = tegra_ivc_write_get_next_frame(channel->ivc);
179 static int tegra_bpmp_wait_master_free(struct tegra_bpmp_channel *channel)
181 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
184 start = ns_to_ktime(local_clock());
187 if (tegra_bpmp_master_free(channel))
190 now = ns_to_ktime(local_clock());
191 } while (ktime_us_delta(now, start) < timeout);
196 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
197 void *data, size_t size)
199 if (data && size > 0)
200 memcpy(data, channel->ib->data, size);
202 return tegra_ivc_read_advance(channel->ivc);
205 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
206 void *data, size_t size)
208 struct tegra_bpmp *bpmp = channel->bpmp;
213 index = tegra_bpmp_channel_get_thread_index(channel);
219 spin_lock_irqsave(&bpmp->lock, flags);
220 err = __tegra_bpmp_channel_read(channel, data, size);
221 clear_bit(index, bpmp->threaded.allocated);
222 spin_unlock_irqrestore(&bpmp->lock, flags);
225 up(&bpmp->threaded.lock);
230 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
231 unsigned int mrq, unsigned long flags,
232 const void *data, size_t size)
234 channel->ob->code = mrq;
235 channel->ob->flags = flags;
237 if (data && size > 0)
238 memcpy(channel->ob->data, data, size);
240 return tegra_ivc_write_advance(channel->ivc);
243 static struct tegra_bpmp_channel *
244 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
245 const void *data, size_t size)
247 unsigned long timeout = bpmp->soc->channels.thread.timeout;
248 unsigned int count = bpmp->soc->channels.thread.count;
249 struct tegra_bpmp_channel *channel;
254 err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
258 spin_lock_irqsave(&bpmp->lock, flags);
260 index = find_first_zero_bit(bpmp->threaded.allocated, count);
261 if (index == count) {
266 channel = tegra_bpmp_channel_get_thread(bpmp, index);
272 if (!tegra_bpmp_master_free(channel)) {
277 set_bit(index, bpmp->threaded.allocated);
279 err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
282 goto clear_allocated;
284 set_bit(index, bpmp->threaded.busy);
286 spin_unlock_irqrestore(&bpmp->lock, flags);
290 clear_bit(index, bpmp->threaded.allocated);
292 spin_unlock_irqrestore(&bpmp->lock, flags);
293 up(&bpmp->threaded.lock);
298 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
299 unsigned int mrq, unsigned long flags,
300 const void *data, size_t size)
304 err = tegra_bpmp_wait_master_free(channel);
308 return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
311 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
312 struct tegra_bpmp_message *msg)
314 struct tegra_bpmp_channel *channel;
317 if (WARN_ON(!irqs_disabled()))
320 if (!tegra_bpmp_message_valid(msg))
323 channel = tegra_bpmp_channel_get_tx(bpmp);
325 err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
326 msg->tx.data, msg->tx.size);
330 err = mbox_send_message(bpmp->mbox.channel, NULL);
334 mbox_client_txdone(bpmp->mbox.channel, 0);
336 err = tegra_bpmp_wait_ack(channel);
340 return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size);
342 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
344 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
345 struct tegra_bpmp_message *msg)
347 struct tegra_bpmp_channel *channel;
348 unsigned long timeout;
351 if (WARN_ON(irqs_disabled()))
354 if (!tegra_bpmp_message_valid(msg))
357 channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
360 return PTR_ERR(channel);
362 err = mbox_send_message(bpmp->mbox.channel, NULL);
366 mbox_client_txdone(bpmp->mbox.channel, 0);
368 timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
370 err = wait_for_completion_timeout(&channel->completion, timeout);
374 return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size);
376 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
378 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
381 struct tegra_bpmp_mrq *entry;
383 list_for_each_entry(entry, &bpmp->mrqs, list)
384 if (entry->mrq == mrq)
390 static void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel,
391 int code, const void *data, size_t size)
393 unsigned long flags = channel->ib->flags;
394 struct tegra_bpmp *bpmp = channel->bpmp;
395 struct tegra_bpmp_mb_data *frame;
398 if (WARN_ON(size > MSG_DATA_MIN_SZ))
401 err = tegra_ivc_read_advance(channel->ivc);
402 if (WARN_ON(err < 0))
405 if ((flags & MSG_ACK) == 0)
408 frame = tegra_ivc_write_get_next_frame(channel->ivc);
409 if (WARN_ON(IS_ERR(frame)))
414 if (data && size > 0)
415 memcpy(frame->data, data, size);
417 err = tegra_ivc_write_advance(channel->ivc);
418 if (WARN_ON(err < 0))
421 if (flags & MSG_RING) {
422 err = mbox_send_message(bpmp->mbox.channel, NULL);
423 if (WARN_ON(err < 0))
426 mbox_client_txdone(bpmp->mbox.channel, 0);
430 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
432 struct tegra_bpmp_channel *channel)
434 struct tegra_bpmp_mrq *entry;
437 spin_lock(&bpmp->lock);
439 entry = tegra_bpmp_find_mrq(bpmp, mrq);
441 spin_unlock(&bpmp->lock);
442 tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
446 entry->handler(mrq, channel, entry->data);
448 spin_unlock(&bpmp->lock);
451 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
452 tegra_bpmp_mrq_handler_t handler, void *data)
454 struct tegra_bpmp_mrq *entry;
460 entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
464 spin_lock_irqsave(&bpmp->lock, flags);
467 entry->handler = handler;
469 list_add(&entry->list, &bpmp->mrqs);
471 spin_unlock_irqrestore(&bpmp->lock, flags);
475 EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
477 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
479 struct tegra_bpmp_mrq *entry;
482 spin_lock_irqsave(&bpmp->lock, flags);
484 entry = tegra_bpmp_find_mrq(bpmp, mrq);
488 list_del(&entry->list);
489 devm_kfree(bpmp->dev, entry);
492 spin_unlock_irqrestore(&bpmp->lock, flags);
494 EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
496 static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
497 struct tegra_bpmp_channel *channel,
500 struct mrq_ping_request *request;
501 struct mrq_ping_response response;
503 request = (struct mrq_ping_request *)channel->ib->data;
505 memset(&response, 0, sizeof(response));
506 response.reply = request->challenge << 1;
508 tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
511 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
513 struct mrq_ping_response response;
514 struct mrq_ping_request request;
515 struct tegra_bpmp_message msg;
520 memset(&request, 0, sizeof(request));
521 request.challenge = 1;
523 memset(&response, 0, sizeof(response));
525 memset(&msg, 0, sizeof(msg));
527 msg.tx.data = &request;
528 msg.tx.size = sizeof(request);
529 msg.rx.data = &response;
530 msg.rx.size = sizeof(response);
532 local_irq_save(flags);
534 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
536 local_irq_restore(flags);
540 "ping ok: challenge: %u, response: %u, time: %lld\n",
541 request.challenge, response.reply,
542 ktime_to_us(ktime_sub(end, start)));
547 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
550 struct mrq_query_tag_request request;
551 struct tegra_bpmp_message msg;
557 virt = dma_alloc_coherent(bpmp->dev, MSG_DATA_MIN_SZ, &phys,
558 GFP_KERNEL | GFP_DMA32);
562 memset(&request, 0, sizeof(request));
565 memset(&msg, 0, sizeof(msg));
566 msg.mrq = MRQ_QUERY_TAG;
567 msg.tx.data = &request;
568 msg.tx.size = sizeof(request);
570 local_irq_save(flags);
571 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
572 local_irq_restore(flags);
575 strlcpy(tag, virt, size);
577 dma_free_coherent(bpmp->dev, MSG_DATA_MIN_SZ, virt, phys);
582 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
584 unsigned long flags = channel->ob->flags;
586 if ((flags & MSG_RING) == 0)
589 complete(&channel->completion);
592 static void tegra_bpmp_handle_rx(struct mbox_client *client, void *data)
594 struct tegra_bpmp *bpmp = mbox_client_to_bpmp(client);
595 struct tegra_bpmp_channel *channel;
596 unsigned int i, count;
599 channel = tegra_bpmp_channel_get_rx(bpmp);
600 count = bpmp->soc->channels.thread.count;
601 busy = bpmp->threaded.busy;
603 if (tegra_bpmp_master_acked(channel))
604 tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);
606 spin_lock(&bpmp->lock);
608 for_each_set_bit(i, busy, count) {
609 struct tegra_bpmp_channel *channel;
611 channel = tegra_bpmp_channel_get_thread(bpmp, i);
615 if (tegra_bpmp_master_acked(channel)) {
616 tegra_bpmp_channel_signal(channel);
621 spin_unlock(&bpmp->lock);
624 static void tegra_bpmp_ivc_notify(struct tegra_ivc *ivc, void *data)
626 struct tegra_bpmp *bpmp = data;
629 if (WARN_ON(bpmp->mbox.channel == NULL))
632 err = mbox_send_message(bpmp->mbox.channel, NULL);
636 mbox_client_txdone(bpmp->mbox.channel, 0);
639 static int tegra_bpmp_channel_init(struct tegra_bpmp_channel *channel,
640 struct tegra_bpmp *bpmp,
643 size_t message_size, queue_size;
647 channel->ivc = devm_kzalloc(bpmp->dev, sizeof(*channel->ivc),
652 message_size = tegra_ivc_align(MSG_MIN_SZ);
653 queue_size = tegra_ivc_total_queue_size(message_size);
654 offset = queue_size * index;
656 err = tegra_ivc_init(channel->ivc, NULL,
657 bpmp->rx.virt + offset, bpmp->rx.phys + offset,
658 bpmp->tx.virt + offset, bpmp->tx.phys + offset,
659 1, message_size, tegra_bpmp_ivc_notify,
662 dev_err(bpmp->dev, "failed to setup IVC for channel %u: %d\n",
667 init_completion(&channel->completion);
668 channel->bpmp = bpmp;
673 static void tegra_bpmp_channel_reset(struct tegra_bpmp_channel *channel)
675 /* reset the channel state */
676 tegra_ivc_reset(channel->ivc);
678 /* sync the channel state with BPMP */
679 while (tegra_ivc_notified(channel->ivc))
683 static void tegra_bpmp_channel_cleanup(struct tegra_bpmp_channel *channel)
685 tegra_ivc_cleanup(channel->ivc);
688 static int tegra_bpmp_probe(struct platform_device *pdev)
690 struct tegra_bpmp_channel *channel;
691 struct tegra_bpmp *bpmp;
697 bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
701 bpmp->soc = of_device_get_match_data(&pdev->dev);
702 bpmp->dev = &pdev->dev;
704 bpmp->tx.pool = of_gen_pool_get(pdev->dev.of_node, "shmem", 0);
705 if (!bpmp->tx.pool) {
706 dev_err(&pdev->dev, "TX shmem pool not found\n");
710 bpmp->tx.virt = gen_pool_dma_alloc(bpmp->tx.pool, 4096, &bpmp->tx.phys);
711 if (!bpmp->tx.virt) {
712 dev_err(&pdev->dev, "failed to allocate from TX pool\n");
716 bpmp->rx.pool = of_gen_pool_get(pdev->dev.of_node, "shmem", 1);
717 if (!bpmp->rx.pool) {
718 dev_err(&pdev->dev, "RX shmem pool not found\n");
723 bpmp->rx.virt = gen_pool_dma_alloc(bpmp->rx.pool, 4096, &bpmp->rx.phys);
724 if (!bpmp->rx.pool) {
725 dev_err(&pdev->dev, "failed to allocate from RX pool\n");
730 INIT_LIST_HEAD(&bpmp->mrqs);
731 spin_lock_init(&bpmp->lock);
733 bpmp->threaded.count = bpmp->soc->channels.thread.count;
734 sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
736 size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
738 bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
739 if (!bpmp->threaded.allocated) {
744 bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
745 if (!bpmp->threaded.busy) {
750 bpmp->num_channels = bpmp->soc->channels.cpu_tx.count +
751 bpmp->soc->channels.thread.count +
752 bpmp->soc->channels.cpu_rx.count;
754 bpmp->channels = devm_kcalloc(&pdev->dev, bpmp->num_channels,
755 sizeof(*channel), GFP_KERNEL);
756 if (!bpmp->channels) {
761 /* message channel initialization */
762 for (i = 0; i < bpmp->num_channels; i++) {
763 struct tegra_bpmp_channel *channel = &bpmp->channels[i];
765 err = tegra_bpmp_channel_init(channel, bpmp, i);
767 goto cleanup_channels;
770 /* mbox registration */
771 bpmp->mbox.client.dev = &pdev->dev;
772 bpmp->mbox.client.rx_callback = tegra_bpmp_handle_rx;
773 bpmp->mbox.client.tx_block = false;
774 bpmp->mbox.client.knows_txdone = false;
776 bpmp->mbox.channel = mbox_request_channel(&bpmp->mbox.client, 0);
777 if (IS_ERR(bpmp->mbox.channel)) {
778 err = PTR_ERR(bpmp->mbox.channel);
779 dev_err(&pdev->dev, "failed to get HSP mailbox: %d\n", err);
780 goto cleanup_channels;
783 /* reset message channels */
784 for (i = 0; i < bpmp->num_channels; i++) {
785 struct tegra_bpmp_channel *channel = &bpmp->channels[i];
787 tegra_bpmp_channel_reset(channel);
790 err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
791 tegra_bpmp_mrq_handle_ping, bpmp);
795 err = tegra_bpmp_ping(bpmp);
797 dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
801 err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag) - 1);
803 dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
807 dev_info(&pdev->dev, "firmware: %s\n", tag);
809 platform_set_drvdata(pdev, bpmp);
811 err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
815 err = tegra_bpmp_init_clocks(bpmp);
819 err = tegra_bpmp_init_resets(bpmp);
823 err = tegra_bpmp_init_powergates(bpmp);
830 tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
832 mbox_free_channel(bpmp->mbox.channel);
835 tegra_bpmp_channel_cleanup(&bpmp->channels[i]);
837 gen_pool_free(bpmp->rx.pool, (unsigned long)bpmp->rx.virt, 4096);
839 gen_pool_free(bpmp->tx.pool, (unsigned long)bpmp->tx.virt, 4096);
843 static const struct tegra_bpmp_soc tegra186_soc = {
848 .timeout = 60 * USEC_PER_SEC,
853 .timeout = 600 * USEC_PER_SEC,
864 static const struct of_device_id tegra_bpmp_match[] = {
865 { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
869 static struct platform_driver tegra_bpmp_driver = {
871 .name = "tegra-bpmp",
872 .of_match_table = tegra_bpmp_match,
874 .probe = tegra_bpmp_probe,
877 static int __init tegra_bpmp_init(void)
879 return platform_driver_register(&tegra_bpmp_driver);
881 core_initcall(tegra_bpmp_init);