GNU Linux-libre 6.9.1-gnu
[releases.git] / drivers / virt / acrn / ioreq.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * ACRN_HSM: Handle I/O requests
4  *
5  * Copyright (C) 2020 Intel Corporation. All rights reserved.
6  *
7  * Authors:
8  *      Jason Chen CJ <jason.cj.chen@intel.com>
9  *      Fengwei Yin <fengwei.yin@intel.com>
10  */
11
12 #include <linux/interrupt.h>
13 #include <linux/io.h>
14 #include <linux/kthread.h>
15 #include <linux/mm.h>
16 #include <linux/slab.h>
17
18 #include <asm/acrn.h>
19
20 #include "acrn_drv.h"
21
22 static void ioreq_pause(void);
23 static void ioreq_resume(void);
24
25 static void ioreq_dispatcher(struct work_struct *work);
26 static struct workqueue_struct *ioreq_wq;
27 static DECLARE_WORK(ioreq_work, ioreq_dispatcher);
28
29 static inline bool has_pending_request(struct acrn_ioreq_client *client)
30 {
31         return !bitmap_empty(client->ioreqs_map, ACRN_IO_REQUEST_MAX);
32 }
33
34 static inline bool is_destroying(struct acrn_ioreq_client *client)
35 {
36         return test_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags);
37 }
38
39 static int ioreq_complete_request(struct acrn_vm *vm, u16 vcpu,
40                                   struct acrn_io_request *acrn_req)
41 {
42         bool polling_mode;
43         int ret = 0;
44
45         polling_mode = acrn_req->completion_polling;
46         /* Add barrier() to make sure the writes are done before completion */
47         smp_store_release(&acrn_req->processed, ACRN_IOREQ_STATE_COMPLETE);
48
49         /*
50          * To fulfill the requirement of real-time in several industry
51          * scenarios, like automotive, ACRN can run under the partition mode,
52          * in which User VMs and Service VM are bound to dedicated CPU cores.
53          * Polling mode of handling the I/O request is introduced to achieve a
54          * faster I/O request handling. In polling mode, the hypervisor polls
55          * I/O request's completion. Once an I/O request is marked as
56          * ACRN_IOREQ_STATE_COMPLETE, hypervisor resumes from the polling point
57          * to continue the I/O request flow. Thus, the completion notification
58          * from HSM of I/O request is not needed.  Please note,
59          * completion_polling needs to be read before the I/O request being
60          * marked as ACRN_IOREQ_STATE_COMPLETE to avoid racing with the
61          * hypervisor.
62          */
63         if (!polling_mode) {
64                 ret = hcall_notify_req_finish(vm->vmid, vcpu);
65                 if (ret < 0)
66                         dev_err(acrn_dev.this_device,
67                                 "Notify I/O request finished failed!\n");
68         }
69
70         return ret;
71 }
72
73 static int acrn_ioreq_complete_request(struct acrn_ioreq_client *client,
74                                        u16 vcpu,
75                                        struct acrn_io_request *acrn_req)
76 {
77         int ret;
78
79         if (vcpu >= client->vm->vcpu_num)
80                 return -EINVAL;
81
82         clear_bit(vcpu, client->ioreqs_map);
83         if (!acrn_req) {
84                 acrn_req = (struct acrn_io_request *)client->vm->ioreq_buf;
85                 acrn_req += vcpu;
86         }
87
88         ret = ioreq_complete_request(client->vm, vcpu, acrn_req);
89
90         return ret;
91 }
92
93 int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu)
94 {
95         int ret = 0;
96
97         spin_lock_bh(&vm->ioreq_clients_lock);
98         if (vm->default_client)
99                 ret = acrn_ioreq_complete_request(vm->default_client,
100                                                   vcpu, NULL);
101         spin_unlock_bh(&vm->ioreq_clients_lock);
102
103         return ret;
104 }
105
106 /**
107  * acrn_ioreq_range_add() - Add an iorange monitored by an ioreq client
108  * @client:     The ioreq client
109  * @type:       Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO)
110  * @start:      Start address of iorange
111  * @end:        End address of iorange
112  *
113  * Return: 0 on success, <0 on error
114  */
115 int acrn_ioreq_range_add(struct acrn_ioreq_client *client,
116                          u32 type, u64 start, u64 end)
117 {
118         struct acrn_ioreq_range *range;
119
120         if (end < start) {
121                 dev_err(acrn_dev.this_device,
122                         "Invalid IO range [0x%llx,0x%llx]\n", start, end);
123                 return -EINVAL;
124         }
125
126         range = kzalloc(sizeof(*range), GFP_KERNEL);
127         if (!range)
128                 return -ENOMEM;
129
130         range->type = type;
131         range->start = start;
132         range->end = end;
133
134         write_lock_bh(&client->range_lock);
135         list_add(&range->list, &client->range_list);
136         write_unlock_bh(&client->range_lock);
137
138         return 0;
139 }
140
141 /**
142  * acrn_ioreq_range_del() - Del an iorange monitored by an ioreq client
143  * @client:     The ioreq client
144  * @type:       Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO)
145  * @start:      Start address of iorange
146  * @end:        End address of iorange
147  */
148 void acrn_ioreq_range_del(struct acrn_ioreq_client *client,
149                           u32 type, u64 start, u64 end)
150 {
151         struct acrn_ioreq_range *range;
152
153         write_lock_bh(&client->range_lock);
154         list_for_each_entry(range, &client->range_list, list) {
155                 if (type == range->type &&
156                     start == range->start &&
157                     end == range->end) {
158                         list_del(&range->list);
159                         kfree(range);
160                         break;
161                 }
162         }
163         write_unlock_bh(&client->range_lock);
164 }
165
166 /*
167  * ioreq_task() is the execution entity of handler thread of an I/O client.
168  * The handler callback of the I/O client is called within the handler thread.
169  */
170 static int ioreq_task(void *data)
171 {
172         struct acrn_ioreq_client *client = data;
173         struct acrn_io_request *req;
174         unsigned long *ioreqs_map;
175         int vcpu, ret;
176
177         /*
178          * Lockless access to ioreqs_map is safe, because
179          * 1) set_bit() and clear_bit() are atomic operations.
180          * 2) I/O requests arrives serialized. The access flow of ioreqs_map is:
181          *      set_bit() - in ioreq_work handler
182          *      Handler callback handles corresponding I/O request
183          *      clear_bit() - in handler thread (include ACRN userspace)
184          *      Mark corresponding I/O request completed
185          *      Loop again if a new I/O request occurs
186          */
187         ioreqs_map = client->ioreqs_map;
188         while (!kthread_should_stop()) {
189                 acrn_ioreq_client_wait(client);
190                 while (has_pending_request(client)) {
191                         vcpu = find_first_bit(ioreqs_map, client->vm->vcpu_num);
192                         req = client->vm->ioreq_buf->req_slot + vcpu;
193                         ret = client->handler(client, req);
194                         if (ret < 0) {
195                                 dev_err(acrn_dev.this_device,
196                                         "IO handle failure: %d\n", ret);
197                                 break;
198                         }
199                         acrn_ioreq_complete_request(client, vcpu, req);
200                 }
201         }
202
203         return 0;
204 }
205
206 /*
207  * For the non-default I/O clients, give them chance to complete the current
208  * I/O requests if there are any. For the default I/O client, it is safe to
209  * clear all pending I/O requests because the clearing request is from ACRN
210  * userspace.
211  */
212 void acrn_ioreq_request_clear(struct acrn_vm *vm)
213 {
214         struct acrn_ioreq_client *client;
215         bool has_pending = false;
216         unsigned long vcpu;
217         int retry = 10;
218
219         /*
220          * IO requests of this VM will be completed directly in
221          * acrn_ioreq_dispatch if ACRN_VM_FLAG_CLEARING_IOREQ flag is set.
222          */
223         set_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags);
224
225         /*
226          * acrn_ioreq_request_clear is only called in VM reset case. Simply
227          * wait 100ms in total for the IO requests' completion.
228          */
229         do {
230                 spin_lock_bh(&vm->ioreq_clients_lock);
231                 list_for_each_entry(client, &vm->ioreq_clients, list) {
232                         has_pending = has_pending_request(client);
233                         if (has_pending)
234                                 break;
235                 }
236                 spin_unlock_bh(&vm->ioreq_clients_lock);
237
238                 if (has_pending)
239                         schedule_timeout_interruptible(HZ / 100);
240         } while (has_pending && --retry > 0);
241         if (retry == 0)
242                 dev_warn(acrn_dev.this_device,
243                          "%s cannot flush pending request!\n", client->name);
244
245         /* Clear all ioreqs belonging to the default client */
246         spin_lock_bh(&vm->ioreq_clients_lock);
247         client = vm->default_client;
248         if (client) {
249                 for_each_set_bit(vcpu, client->ioreqs_map, ACRN_IO_REQUEST_MAX)
250                         acrn_ioreq_complete_request(client, vcpu, NULL);
251         }
252         spin_unlock_bh(&vm->ioreq_clients_lock);
253
254         /* Clear ACRN_VM_FLAG_CLEARING_IOREQ flag after the clearing */
255         clear_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags);
256 }
257
258 int acrn_ioreq_client_wait(struct acrn_ioreq_client *client)
259 {
260         if (client->is_default) {
261                 /*
262                  * In the default client, a user space thread waits on the
263                  * waitqueue. The is_destroying() check is used to notify user
264                  * space the client is going to be destroyed.
265                  */
266                 wait_event_interruptible(client->wq,
267                                          has_pending_request(client) ||
268                                          is_destroying(client));
269                 if (is_destroying(client))
270                         return -ENODEV;
271         } else {
272                 wait_event_interruptible(client->wq,
273                                          has_pending_request(client) ||
274                                          kthread_should_stop());
275         }
276
277         return 0;
278 }
279
280 static bool is_cfg_addr(struct acrn_io_request *req)
281 {
282         return ((req->type == ACRN_IOREQ_TYPE_PORTIO) &&
283                 (req->reqs.pio_request.address == 0xcf8));
284 }
285
286 static bool is_cfg_data(struct acrn_io_request *req)
287 {
288         return ((req->type == ACRN_IOREQ_TYPE_PORTIO) &&
289                 ((req->reqs.pio_request.address >= 0xcfc) &&
290                  (req->reqs.pio_request.address < (0xcfc + 4))));
291 }
292
293 /* The low 8-bit of supported pci_reg addr.*/
294 #define PCI_LOWREG_MASK  0xFC
295 /* The high 4-bit of supported pci_reg addr */
296 #define PCI_HIGHREG_MASK 0xF00
297 /* Max number of supported functions */
298 #define PCI_FUNCMAX     7
299 /* Max number of supported slots */
300 #define PCI_SLOTMAX     31
301 /* Max number of supported buses */
302 #define PCI_BUSMAX      255
303 #define CONF1_ENABLE    0x80000000UL
304 /*
305  * A PCI configuration space access via PIO 0xCF8 and 0xCFC normally has two
306  * following steps:
307  *   1) writes address into 0xCF8 port
308  *   2) accesses data in/from 0xCFC
309  * This function combines such paired PCI configuration space I/O requests into
310  * one ACRN_IOREQ_TYPE_PCICFG type I/O request and continues the processing.
311  */
312 static bool handle_cf8cfc(struct acrn_vm *vm,
313                           struct acrn_io_request *req, u16 vcpu)
314 {
315         int offset, pci_cfg_addr, pci_reg;
316         bool is_handled = false;
317
318         if (is_cfg_addr(req)) {
319                 WARN_ON(req->reqs.pio_request.size != 4);
320                 if (req->reqs.pio_request.direction == ACRN_IOREQ_DIR_WRITE)
321                         vm->pci_conf_addr = req->reqs.pio_request.value;
322                 else
323                         req->reqs.pio_request.value = vm->pci_conf_addr;
324                 is_handled = true;
325         } else if (is_cfg_data(req)) {
326                 if (!(vm->pci_conf_addr & CONF1_ENABLE)) {
327                         if (req->reqs.pio_request.direction ==
328                                         ACRN_IOREQ_DIR_READ)
329                                 req->reqs.pio_request.value = 0xffffffff;
330                         is_handled = true;
331                 } else {
332                         offset = req->reqs.pio_request.address - 0xcfc;
333
334                         req->type = ACRN_IOREQ_TYPE_PCICFG;
335                         pci_cfg_addr = vm->pci_conf_addr;
336                         req->reqs.pci_request.bus =
337                                         (pci_cfg_addr >> 16) & PCI_BUSMAX;
338                         req->reqs.pci_request.dev =
339                                         (pci_cfg_addr >> 11) & PCI_SLOTMAX;
340                         req->reqs.pci_request.func =
341                                         (pci_cfg_addr >> 8) & PCI_FUNCMAX;
342                         pci_reg = (pci_cfg_addr & PCI_LOWREG_MASK) +
343                                    ((pci_cfg_addr >> 16) & PCI_HIGHREG_MASK);
344                         req->reqs.pci_request.reg = pci_reg + offset;
345                 }
346         }
347
348         if (is_handled)
349                 ioreq_complete_request(vm, vcpu, req);
350
351         return is_handled;
352 }
353
354 static bool acrn_in_range(struct acrn_ioreq_range *range,
355                      struct acrn_io_request *req)
356 {
357         bool ret = false;
358
359         if (range->type == req->type) {
360                 switch (req->type) {
361                 case ACRN_IOREQ_TYPE_MMIO:
362                         if (req->reqs.mmio_request.address >= range->start &&
363                             (req->reqs.mmio_request.address +
364                              req->reqs.mmio_request.size - 1) <= range->end)
365                                 ret = true;
366                         break;
367                 case ACRN_IOREQ_TYPE_PORTIO:
368                         if (req->reqs.pio_request.address >= range->start &&
369                             (req->reqs.pio_request.address +
370                              req->reqs.pio_request.size - 1) <= range->end)
371                                 ret = true;
372                         break;
373                 default:
374                         break;
375                 }
376         }
377
378         return ret;
379 }
380
381 static struct acrn_ioreq_client *find_ioreq_client(struct acrn_vm *vm,
382                                                    struct acrn_io_request *req)
383 {
384         struct acrn_ioreq_client *client, *found = NULL;
385         struct acrn_ioreq_range *range;
386
387         lockdep_assert_held(&vm->ioreq_clients_lock);
388
389         list_for_each_entry(client, &vm->ioreq_clients, list) {
390                 read_lock_bh(&client->range_lock);
391                 list_for_each_entry(range, &client->range_list, list) {
392                         if (acrn_in_range(range, req)) {
393                                 found = client;
394                                 break;
395                         }
396                 }
397                 read_unlock_bh(&client->range_lock);
398                 if (found)
399                         break;
400         }
401         return found ? found : vm->default_client;
402 }
403
404 /**
405  * acrn_ioreq_client_create() - Create an ioreq client
406  * @vm:         The VM that this client belongs to
407  * @handler:    The ioreq_handler of ioreq client acrn_hsm will create a kernel
408  *              thread and call the handler to handle I/O requests.
409  * @priv:       Private data for the handler
410  * @is_default: If it is the default client
411  * @name:       The name of ioreq client
412  *
413  * Return: acrn_ioreq_client pointer on success, NULL on error
414  */
415 struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm,
416                                                    ioreq_handler_t handler,
417                                                    void *priv, bool is_default,
418                                                    const char *name)
419 {
420         struct acrn_ioreq_client *client;
421
422         if (!handler && !is_default) {
423                 dev_dbg(acrn_dev.this_device,
424                         "Cannot create non-default client w/o handler!\n");
425                 return NULL;
426         }
427         client = kzalloc(sizeof(*client), GFP_KERNEL);
428         if (!client)
429                 return NULL;
430
431         client->handler = handler;
432         client->vm = vm;
433         client->priv = priv;
434         client->is_default = is_default;
435         if (name)
436                 strncpy(client->name, name, sizeof(client->name) - 1);
437         rwlock_init(&client->range_lock);
438         INIT_LIST_HEAD(&client->range_list);
439         init_waitqueue_head(&client->wq);
440
441         if (client->handler) {
442                 client->thread = kthread_run(ioreq_task, client, "VM%u-%s",
443                                              client->vm->vmid, client->name);
444                 if (IS_ERR(client->thread)) {
445                         kfree(client);
446                         return NULL;
447                 }
448         }
449
450         spin_lock_bh(&vm->ioreq_clients_lock);
451         if (is_default)
452                 vm->default_client = client;
453         else
454                 list_add(&client->list, &vm->ioreq_clients);
455         spin_unlock_bh(&vm->ioreq_clients_lock);
456
457         dev_dbg(acrn_dev.this_device, "Created ioreq client %s.\n", name);
458         return client;
459 }
460
461 /**
462  * acrn_ioreq_client_destroy() - Destroy an ioreq client
463  * @client:     The ioreq client
464  */
465 void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client)
466 {
467         struct acrn_ioreq_range *range, *next;
468         struct acrn_vm *vm = client->vm;
469
470         dev_dbg(acrn_dev.this_device,
471                 "Destroy ioreq client %s.\n", client->name);
472         ioreq_pause();
473         set_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags);
474         if (client->is_default)
475                 wake_up_interruptible(&client->wq);
476         else
477                 kthread_stop(client->thread);
478
479         spin_lock_bh(&vm->ioreq_clients_lock);
480         if (client->is_default)
481                 vm->default_client = NULL;
482         else
483                 list_del(&client->list);
484         spin_unlock_bh(&vm->ioreq_clients_lock);
485
486         write_lock_bh(&client->range_lock);
487         list_for_each_entry_safe(range, next, &client->range_list, list) {
488                 list_del(&range->list);
489                 kfree(range);
490         }
491         write_unlock_bh(&client->range_lock);
492         kfree(client);
493
494         ioreq_resume();
495 }
496
497 static int acrn_ioreq_dispatch(struct acrn_vm *vm)
498 {
499         struct acrn_ioreq_client *client;
500         struct acrn_io_request *req;
501         int i;
502
503         for (i = 0; i < vm->vcpu_num; i++) {
504                 req = vm->ioreq_buf->req_slot + i;
505
506                 /* barrier the read of processed of acrn_io_request */
507                 if (smp_load_acquire(&req->processed) ==
508                                      ACRN_IOREQ_STATE_PENDING) {
509                         /* Complete the IO request directly in clearing stage */
510                         if (test_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags)) {
511                                 ioreq_complete_request(vm, i, req);
512                                 continue;
513                         }
514                         if (handle_cf8cfc(vm, req, i))
515                                 continue;
516
517                         spin_lock_bh(&vm->ioreq_clients_lock);
518                         client = find_ioreq_client(vm, req);
519                         if (!client) {
520                                 dev_err(acrn_dev.this_device,
521                                         "Failed to find ioreq client!\n");
522                                 spin_unlock_bh(&vm->ioreq_clients_lock);
523                                 return -EINVAL;
524                         }
525                         if (!client->is_default)
526                                 req->kernel_handled = 1;
527                         else
528                                 req->kernel_handled = 0;
529                         /*
530                          * Add barrier() to make sure the writes are done
531                          * before setting ACRN_IOREQ_STATE_PROCESSING
532                          */
533                         smp_store_release(&req->processed,
534                                           ACRN_IOREQ_STATE_PROCESSING);
535                         set_bit(i, client->ioreqs_map);
536                         wake_up_interruptible(&client->wq);
537                         spin_unlock_bh(&vm->ioreq_clients_lock);
538                 }
539         }
540
541         return 0;
542 }
543
544 static void ioreq_dispatcher(struct work_struct *work)
545 {
546         struct acrn_vm *vm;
547
548         read_lock(&acrn_vm_list_lock);
549         list_for_each_entry(vm, &acrn_vm_list, list) {
550                 if (!vm->ioreq_buf)
551                         break;
552                 acrn_ioreq_dispatch(vm);
553         }
554         read_unlock(&acrn_vm_list_lock);
555 }
556
557 static void ioreq_intr_handler(void)
558 {
559         queue_work(ioreq_wq, &ioreq_work);
560 }
561
562 static void ioreq_pause(void)
563 {
564         /* Flush and unarm the handler to ensure no I/O requests pending */
565         acrn_remove_intr_handler();
566         drain_workqueue(ioreq_wq);
567 }
568
569 static void ioreq_resume(void)
570 {
571         /* Schedule after enabling in case other clients miss interrupt */
572         acrn_setup_intr_handler(ioreq_intr_handler);
573         queue_work(ioreq_wq, &ioreq_work);
574 }
575
576 int acrn_ioreq_intr_setup(void)
577 {
578         acrn_setup_intr_handler(ioreq_intr_handler);
579         ioreq_wq = alloc_ordered_workqueue("ioreq_wq",
580                                            WQ_HIGHPRI | WQ_MEM_RECLAIM);
581         if (!ioreq_wq) {
582                 dev_err(acrn_dev.this_device, "Failed to alloc workqueue!\n");
583                 acrn_remove_intr_handler();
584                 return -ENOMEM;
585         }
586         return 0;
587 }
588
589 void acrn_ioreq_intr_remove(void)
590 {
591         if (ioreq_wq)
592                 destroy_workqueue(ioreq_wq);
593         acrn_remove_intr_handler();
594 }
595
596 int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma)
597 {
598         struct acrn_ioreq_buffer *set_buffer;
599         struct page *page;
600         int ret;
601
602         if (vm->ioreq_buf)
603                 return -EEXIST;
604
605         set_buffer = kzalloc(sizeof(*set_buffer), GFP_KERNEL);
606         if (!set_buffer)
607                 return -ENOMEM;
608
609         ret = pin_user_pages_fast(buf_vma, 1,
610                                   FOLL_WRITE | FOLL_LONGTERM, &page);
611         if (unlikely(ret != 1) || !page) {
612                 dev_err(acrn_dev.this_device, "Failed to pin ioreq page!\n");
613                 ret = -EFAULT;
614                 goto free_buf;
615         }
616
617         vm->ioreq_buf = page_address(page);
618         vm->ioreq_page = page;
619         set_buffer->ioreq_buf = page_to_phys(page);
620         ret = hcall_set_ioreq_buffer(vm->vmid, virt_to_phys(set_buffer));
621         if (ret < 0) {
622                 dev_err(acrn_dev.this_device, "Failed to init ioreq buffer!\n");
623                 unpin_user_page(page);
624                 vm->ioreq_buf = NULL;
625                 goto free_buf;
626         }
627
628         dev_dbg(acrn_dev.this_device,
629                 "Init ioreq buffer %pK!\n", vm->ioreq_buf);
630         ret = 0;
631 free_buf:
632         kfree(set_buffer);
633         return ret;
634 }
635
636 void acrn_ioreq_deinit(struct acrn_vm *vm)
637 {
638         struct acrn_ioreq_client *client, *next;
639
640         dev_dbg(acrn_dev.this_device,
641                 "Deinit ioreq buffer %pK!\n", vm->ioreq_buf);
642         /* Destroy all clients belonging to this VM */
643         list_for_each_entry_safe(client, next, &vm->ioreq_clients, list)
644                 acrn_ioreq_client_destroy(client);
645         if (vm->default_client)
646                 acrn_ioreq_client_destroy(vm->default_client);
647
648         if (vm->ioreq_buf && vm->ioreq_page) {
649                 unpin_user_page(vm->ioreq_page);
650                 vm->ioreq_buf = NULL;
651         }
652 }