GNU Linux-libre 6.8.9-gnu
[releases.git] / drivers / vdpa / vdpa_sim / vdpa_sim.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VDPA device simulator core.
4  *
5  * Copyright (c) 2020, Red Hat Inc. All rights reserved.
6  *     Author: Jason Wang <jasowang@redhat.com>
7  *
8  */
9
10 #include <linux/init.h>
11 #include <linux/module.h>
12 #include <linux/device.h>
13 #include <linux/kernel.h>
14 #include <linux/kthread.h>
15 #include <linux/slab.h>
16 #include <linux/dma-map-ops.h>
17 #include <linux/vringh.h>
18 #include <linux/vdpa.h>
19 #include <linux/vhost_iotlb.h>
20 #include <uapi/linux/vdpa.h>
21 #include <uapi/linux/vhost_types.h>
22
23 #include "vdpa_sim.h"
24
25 #define DRV_VERSION  "0.1"
26 #define DRV_AUTHOR   "Jason Wang <jasowang@redhat.com>"
27 #define DRV_DESC     "vDPA Device Simulator core"
28 #define DRV_LICENSE  "GPL v2"
29
30 static int batch_mapping = 1;
31 module_param(batch_mapping, int, 0444);
32 MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable");
33
34 static int max_iotlb_entries = 2048;
35 module_param(max_iotlb_entries, int, 0444);
36 MODULE_PARM_DESC(max_iotlb_entries,
37                  "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");
38
39 static bool use_va = true;
40 module_param(use_va, bool, 0444);
41 MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA");
42
43 #define VDPASIM_QUEUE_ALIGN PAGE_SIZE
44 #define VDPASIM_QUEUE_MAX 256
45 #define VDPASIM_VENDOR_ID 0
46
47 struct vdpasim_mm_work {
48         struct kthread_work work;
49         struct vdpasim *vdpasim;
50         struct mm_struct *mm_to_bind;
51         int ret;
52 };
53
54 static void vdpasim_mm_work_fn(struct kthread_work *work)
55 {
56         struct vdpasim_mm_work *mm_work =
57                 container_of(work, struct vdpasim_mm_work, work);
58         struct vdpasim *vdpasim = mm_work->vdpasim;
59
60         mm_work->ret = 0;
61
62         //TODO: should we attach the cgroup of the mm owner?
63         vdpasim->mm_bound = mm_work->mm_to_bind;
64 }
65
66 static void vdpasim_worker_change_mm_sync(struct vdpasim *vdpasim,
67                                           struct vdpasim_mm_work *mm_work)
68 {
69         struct kthread_work *work = &mm_work->work;
70
71         kthread_init_work(work, vdpasim_mm_work_fn);
72         kthread_queue_work(vdpasim->worker, work);
73
74         kthread_flush_work(work);
75 }
76
77 static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)
78 {
79         return container_of(vdpa, struct vdpasim, vdpa);
80 }
81
82 static void vdpasim_vq_notify(struct vringh *vring)
83 {
84         struct vdpasim_virtqueue *vq =
85                 container_of(vring, struct vdpasim_virtqueue, vring);
86
87         if (!vq->cb)
88                 return;
89
90         vq->cb(vq->private);
91 }
92
93 static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
94 {
95         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
96         uint16_t last_avail_idx = vq->vring.last_avail_idx;
97         struct vring_desc *desc = (struct vring_desc *)
98                                   (uintptr_t)vq->desc_addr;
99         struct vring_avail *avail = (struct vring_avail *)
100                                     (uintptr_t)vq->driver_addr;
101         struct vring_used *used = (struct vring_used *)
102                                   (uintptr_t)vq->device_addr;
103
104         if (use_va && vdpasim->mm_bound) {
105                 vringh_init_iotlb_va(&vq->vring, vdpasim->features, vq->num,
106                                      true, desc, avail, used);
107         } else {
108                 vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num,
109                                   true, desc, avail, used);
110         }
111
112         vq->vring.last_avail_idx = last_avail_idx;
113
114         /*
115          * Since vdpa_sim does not support receive inflight descriptors as a
116          * destination of a migration, let's set both avail_idx and used_idx
117          * the same at vq start.  This is how vhost-user works in a
118          * VHOST_SET_VRING_BASE call.
119          *
120          * Although the simple fix is to set last_used_idx at
121          * vdpasim_set_vq_state, it would be reset at vdpasim_queue_ready.
122          */
123         vq->vring.last_used_idx = last_avail_idx;
124         vq->vring.notify = vdpasim_vq_notify;
125 }
126
127 static void vdpasim_vq_reset(struct vdpasim *vdpasim,
128                              struct vdpasim_virtqueue *vq)
129 {
130         vq->ready = false;
131         vq->desc_addr = 0;
132         vq->driver_addr = 0;
133         vq->device_addr = 0;
134         vq->cb = NULL;
135         vq->private = NULL;
136         vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features,
137                           VDPASIM_QUEUE_MAX, false, NULL, NULL, NULL);
138
139         vq->vring.notify = NULL;
140 }
141
142 static void vdpasim_do_reset(struct vdpasim *vdpasim, u32 flags)
143 {
144         int i;
145
146         spin_lock(&vdpasim->iommu_lock);
147
148         for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
149                 vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]);
150                 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],
151                                  &vdpasim->iommu_lock);
152         }
153
154         if (flags & VDPA_RESET_F_CLEAN_MAP) {
155                 for (i = 0; i < vdpasim->dev_attr.nas; i++) {
156                         vhost_iotlb_reset(&vdpasim->iommu[i]);
157                         vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX,
158                                               0, VHOST_MAP_RW);
159                         vdpasim->iommu_pt[i] = true;
160                 }
161         }
162
163         vdpasim->running = false;
164         spin_unlock(&vdpasim->iommu_lock);
165
166         vdpasim->features = 0;
167         vdpasim->status = 0;
168         ++vdpasim->generation;
169 }
170
171 static const struct vdpa_config_ops vdpasim_config_ops;
172 static const struct vdpa_config_ops vdpasim_batch_config_ops;
173
174 static void vdpasim_work_fn(struct kthread_work *work)
175 {
176         struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
177         struct mm_struct *mm = vdpasim->mm_bound;
178
179         if (use_va && mm) {
180                 if (!mmget_not_zero(mm))
181                         return;
182                 kthread_use_mm(mm);
183         }
184
185         vdpasim->dev_attr.work_fn(vdpasim);
186
187         if (use_va && mm) {
188                 kthread_unuse_mm(mm);
189                 mmput(mm);
190         }
191 }
192
193 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
194                                const struct vdpa_dev_set_config *config)
195 {
196         const struct vdpa_config_ops *ops;
197         struct vdpa_device *vdpa;
198         struct vdpasim *vdpasim;
199         struct device *dev;
200         int i, ret = -ENOMEM;
201
202         if (!dev_attr->alloc_size)
203                 return ERR_PTR(-EINVAL);
204
205         if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
206                 if (config->device_features &
207                     ~dev_attr->supported_features)
208                         return ERR_PTR(-EINVAL);
209                 dev_attr->supported_features =
210                         config->device_features;
211         }
212
213         if (batch_mapping)
214                 ops = &vdpasim_batch_config_ops;
215         else
216                 ops = &vdpasim_config_ops;
217
218         vdpa = __vdpa_alloc_device(NULL, ops,
219                                    dev_attr->ngroups, dev_attr->nas,
220                                    dev_attr->alloc_size,
221                                    dev_attr->name, use_va);
222         if (IS_ERR(vdpa)) {
223                 ret = PTR_ERR(vdpa);
224                 goto err_alloc;
225         }
226
227         vdpasim = vdpa_to_sim(vdpa);
228         vdpasim->dev_attr = *dev_attr;
229         dev = &vdpasim->vdpa.dev;
230
231         kthread_init_work(&vdpasim->work, vdpasim_work_fn);
232         vdpasim->worker = kthread_create_worker(0, "vDPA sim worker: %s",
233                                                 dev_attr->name);
234         if (IS_ERR(vdpasim->worker))
235                 goto err_iommu;
236
237         mutex_init(&vdpasim->mutex);
238         spin_lock_init(&vdpasim->iommu_lock);
239
240         dev->dma_mask = &dev->coherent_dma_mask;
241         if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
242                 goto err_iommu;
243         vdpasim->vdpa.mdev = dev_attr->mgmt_dev;
244
245         vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL);
246         if (!vdpasim->config)
247                 goto err_iommu;
248
249         vdpasim->vqs = kcalloc(dev_attr->nvqs, sizeof(struct vdpasim_virtqueue),
250                                GFP_KERNEL);
251         if (!vdpasim->vqs)
252                 goto err_iommu;
253
254         vdpasim->iommu = kmalloc_array(vdpasim->dev_attr.nas,
255                                        sizeof(*vdpasim->iommu), GFP_KERNEL);
256         if (!vdpasim->iommu)
257                 goto err_iommu;
258
259         vdpasim->iommu_pt = kmalloc_array(vdpasim->dev_attr.nas,
260                                           sizeof(*vdpasim->iommu_pt), GFP_KERNEL);
261         if (!vdpasim->iommu_pt)
262                 goto err_iommu;
263
264         for (i = 0; i < vdpasim->dev_attr.nas; i++) {
265                 vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);
266                 vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 0,
267                                       VHOST_MAP_RW);
268                 vdpasim->iommu_pt[i] = true;
269         }
270
271         for (i = 0; i < dev_attr->nvqs; i++)
272                 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],
273                                  &vdpasim->iommu_lock);
274
275         vdpasim->vdpa.dma_dev = dev;
276
277         return vdpasim;
278
279 err_iommu:
280         put_device(dev);
281 err_alloc:
282         return ERR_PTR(ret);
283 }
284 EXPORT_SYMBOL_GPL(vdpasim_create);
285
286 void vdpasim_schedule_work(struct vdpasim *vdpasim)
287 {
288         kthread_queue_work(vdpasim->worker, &vdpasim->work);
289 }
290 EXPORT_SYMBOL_GPL(vdpasim_schedule_work);
291
292 static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx,
293                                   u64 desc_area, u64 driver_area,
294                                   u64 device_area)
295 {
296         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
297         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
298
299         vq->desc_addr = desc_area;
300         vq->driver_addr = driver_area;
301         vq->device_addr = device_area;
302
303         return 0;
304 }
305
306 static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
307 {
308         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
309         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
310
311         vq->num = num;
312 }
313
314 static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
315 {
316         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
317         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
318
319         if (!vdpasim->running &&
320             (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
321                 vdpasim->pending_kick = true;
322                 return;
323         }
324
325         if (vq->ready)
326                 vdpasim_schedule_work(vdpasim);
327 }
328
329 static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
330                               struct vdpa_callback *cb)
331 {
332         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
333         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
334
335         vq->cb = cb->callback;
336         vq->private = cb->private;
337 }
338
339 static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready)
340 {
341         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
342         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
343         bool old_ready;
344
345         mutex_lock(&vdpasim->mutex);
346         old_ready = vq->ready;
347         vq->ready = ready;
348         if (vq->ready && !old_ready) {
349                 vdpasim_queue_ready(vdpasim, idx);
350         }
351         mutex_unlock(&vdpasim->mutex);
352 }
353
354 static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
355 {
356         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
357         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
358
359         return vq->ready;
360 }
361
362 static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,
363                                 const struct vdpa_vq_state *state)
364 {
365         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
366         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
367         struct vringh *vrh = &vq->vring;
368
369         mutex_lock(&vdpasim->mutex);
370         vrh->last_avail_idx = state->split.avail_index;
371         mutex_unlock(&vdpasim->mutex);
372
373         return 0;
374 }
375
376 static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx,
377                                 struct vdpa_vq_state *state)
378 {
379         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
380         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
381         struct vringh *vrh = &vq->vring;
382
383         state->split.avail_index = vrh->last_avail_idx;
384         return 0;
385 }
386
387 static int vdpasim_get_vq_stats(struct vdpa_device *vdpa, u16 idx,
388                                 struct sk_buff *msg,
389                                 struct netlink_ext_ack *extack)
390 {
391         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
392
393         if (vdpasim->dev_attr.get_stats)
394                 return vdpasim->dev_attr.get_stats(vdpasim, idx,
395                                                    msg, extack);
396         return -EOPNOTSUPP;
397 }
398
399 static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa)
400 {
401         return VDPASIM_QUEUE_ALIGN;
402 }
403
404 static u32 vdpasim_get_vq_group(struct vdpa_device *vdpa, u16 idx)
405 {
406         /* RX and TX belongs to group 0, CVQ belongs to group 1 */
407         if (idx == 2)
408                 return 1;
409         else
410                 return 0;
411 }
412
413 static u64 vdpasim_get_device_features(struct vdpa_device *vdpa)
414 {
415         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
416
417         return vdpasim->dev_attr.supported_features;
418 }
419
420 static u64 vdpasim_get_backend_features(const struct vdpa_device *vdpa)
421 {
422         return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
423 }
424
425 static int vdpasim_set_driver_features(struct vdpa_device *vdpa, u64 features)
426 {
427         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
428
429         /* DMA mapping must be done by driver */
430         if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
431                 return -EINVAL;
432
433         vdpasim->features = features & vdpasim->dev_attr.supported_features;
434
435         return 0;
436 }
437
438 static u64 vdpasim_get_driver_features(struct vdpa_device *vdpa)
439 {
440         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
441
442         return vdpasim->features;
443 }
444
445 static void vdpasim_set_config_cb(struct vdpa_device *vdpa,
446                                   struct vdpa_callback *cb)
447 {
448         /* We don't support config interrupt */
449 }
450
451 static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa)
452 {
453         return VDPASIM_QUEUE_MAX;
454 }
455
456 static u32 vdpasim_get_device_id(struct vdpa_device *vdpa)
457 {
458         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
459
460         return vdpasim->dev_attr.id;
461 }
462
463 static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa)
464 {
465         return VDPASIM_VENDOR_ID;
466 }
467
468 static u8 vdpasim_get_status(struct vdpa_device *vdpa)
469 {
470         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
471         u8 status;
472
473         mutex_lock(&vdpasim->mutex);
474         status = vdpasim->status;
475         mutex_unlock(&vdpasim->mutex);
476
477         return status;
478 }
479
480 static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status)
481 {
482         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
483
484         mutex_lock(&vdpasim->mutex);
485         vdpasim->status = status;
486         vdpasim->running = (status & VIRTIO_CONFIG_S_DRIVER_OK) != 0;
487         mutex_unlock(&vdpasim->mutex);
488 }
489
490 static int vdpasim_compat_reset(struct vdpa_device *vdpa, u32 flags)
491 {
492         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
493
494         mutex_lock(&vdpasim->mutex);
495         vdpasim->status = 0;
496         vdpasim_do_reset(vdpasim, flags);
497         mutex_unlock(&vdpasim->mutex);
498
499         return 0;
500 }
501
502 static int vdpasim_reset(struct vdpa_device *vdpa)
503 {
504         return vdpasim_compat_reset(vdpa, 0);
505 }
506
507 static int vdpasim_suspend(struct vdpa_device *vdpa)
508 {
509         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
510
511         mutex_lock(&vdpasim->mutex);
512         vdpasim->running = false;
513         mutex_unlock(&vdpasim->mutex);
514
515         return 0;
516 }
517
518 static int vdpasim_resume(struct vdpa_device *vdpa)
519 {
520         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
521         int i;
522
523         mutex_lock(&vdpasim->mutex);
524         vdpasim->running = true;
525
526         if (vdpasim->pending_kick) {
527                 /* Process pending descriptors */
528                 for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
529                         vdpasim_kick_vq(vdpa, i);
530
531                 vdpasim->pending_kick = false;
532         }
533
534         mutex_unlock(&vdpasim->mutex);
535
536         return 0;
537 }
538
539 static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
540 {
541         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
542
543         return vdpasim->dev_attr.config_size;
544 }
545
546 static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset,
547                              void *buf, unsigned int len)
548 {
549         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
550
551         if (offset + len > vdpasim->dev_attr.config_size)
552                 return;
553
554         if (vdpasim->dev_attr.get_config)
555                 vdpasim->dev_attr.get_config(vdpasim, vdpasim->config);
556
557         memcpy(buf, vdpasim->config + offset, len);
558 }
559
560 static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset,
561                              const void *buf, unsigned int len)
562 {
563         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
564
565         if (offset + len > vdpasim->dev_attr.config_size)
566                 return;
567
568         memcpy(vdpasim->config + offset, buf, len);
569
570         if (vdpasim->dev_attr.set_config)
571                 vdpasim->dev_attr.set_config(vdpasim, vdpasim->config);
572 }
573
574 static u32 vdpasim_get_generation(struct vdpa_device *vdpa)
575 {
576         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
577
578         return vdpasim->generation;
579 }
580
581 static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa)
582 {
583         struct vdpa_iova_range range = {
584                 .first = 0ULL,
585                 .last = ULLONG_MAX,
586         };
587
588         return range;
589 }
590
591 static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group,
592                                   unsigned int asid)
593 {
594         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
595         struct vhost_iotlb *iommu;
596         int i;
597
598         if (group > vdpasim->dev_attr.ngroups)
599                 return -EINVAL;
600
601         if (asid >= vdpasim->dev_attr.nas)
602                 return -EINVAL;
603
604         iommu = &vdpasim->iommu[asid];
605
606         mutex_lock(&vdpasim->mutex);
607
608         for (i = 0; i < vdpasim->dev_attr.nvqs; i++)
609                 if (vdpasim_get_vq_group(vdpa, i) == group)
610                         vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu,
611                                          &vdpasim->iommu_lock);
612
613         mutex_unlock(&vdpasim->mutex);
614
615         return 0;
616 }
617
618 static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid,
619                            struct vhost_iotlb *iotlb)
620 {
621         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
622         struct vhost_iotlb_map *map;
623         struct vhost_iotlb *iommu;
624         u64 start = 0ULL, last = 0ULL - 1;
625         int ret;
626
627         if (asid >= vdpasim->dev_attr.nas)
628                 return -EINVAL;
629
630         spin_lock(&vdpasim->iommu_lock);
631
632         iommu = &vdpasim->iommu[asid];
633         vhost_iotlb_reset(iommu);
634         vdpasim->iommu_pt[asid] = false;
635
636         for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
637              map = vhost_iotlb_itree_next(map, start, last)) {
638                 ret = vhost_iotlb_add_range(iommu, map->start,
639                                             map->last, map->addr, map->perm);
640                 if (ret)
641                         goto err;
642         }
643         spin_unlock(&vdpasim->iommu_lock);
644         return 0;
645
646 err:
647         vhost_iotlb_reset(iommu);
648         spin_unlock(&vdpasim->iommu_lock);
649         return ret;
650 }
651
652 static int vdpasim_reset_map(struct vdpa_device *vdpa, unsigned int asid)
653 {
654         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
655
656         if (asid >= vdpasim->dev_attr.nas)
657                 return -EINVAL;
658
659         spin_lock(&vdpasim->iommu_lock);
660         if (vdpasim->iommu_pt[asid])
661                 goto out;
662         vhost_iotlb_reset(&vdpasim->iommu[asid]);
663         vhost_iotlb_add_range(&vdpasim->iommu[asid], 0, ULONG_MAX,
664                               0, VHOST_MAP_RW);
665         vdpasim->iommu_pt[asid] = true;
666 out:
667         spin_unlock(&vdpasim->iommu_lock);
668         return 0;
669 }
670
671 static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm)
672 {
673         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
674         struct vdpasim_mm_work mm_work;
675
676         mm_work.vdpasim = vdpasim;
677         mm_work.mm_to_bind = mm;
678
679         vdpasim_worker_change_mm_sync(vdpasim, &mm_work);
680
681         return mm_work.ret;
682 }
683
684 static void vdpasim_unbind_mm(struct vdpa_device *vdpa)
685 {
686         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
687         struct vdpasim_mm_work mm_work;
688
689         mm_work.vdpasim = vdpasim;
690         mm_work.mm_to_bind = NULL;
691
692         vdpasim_worker_change_mm_sync(vdpasim, &mm_work);
693 }
694
695 static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid,
696                            u64 iova, u64 size,
697                            u64 pa, u32 perm, void *opaque)
698 {
699         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
700         int ret;
701
702         if (asid >= vdpasim->dev_attr.nas)
703                 return -EINVAL;
704
705         spin_lock(&vdpasim->iommu_lock);
706         if (vdpasim->iommu_pt[asid]) {
707                 vhost_iotlb_reset(&vdpasim->iommu[asid]);
708                 vdpasim->iommu_pt[asid] = false;
709         }
710         ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova,
711                                         iova + size - 1, pa, perm, opaque);
712         spin_unlock(&vdpasim->iommu_lock);
713
714         return ret;
715 }
716
717 static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid,
718                              u64 iova, u64 size)
719 {
720         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
721
722         if (asid >= vdpasim->dev_attr.nas)
723                 return -EINVAL;
724
725         if (vdpasim->iommu_pt[asid]) {
726                 vhost_iotlb_reset(&vdpasim->iommu[asid]);
727                 vdpasim->iommu_pt[asid] = false;
728         }
729
730         spin_lock(&vdpasim->iommu_lock);
731         vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1);
732         spin_unlock(&vdpasim->iommu_lock);
733
734         return 0;
735 }
736
737 static void vdpasim_free(struct vdpa_device *vdpa)
738 {
739         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
740         int i;
741
742         kthread_cancel_work_sync(&vdpasim->work);
743         kthread_destroy_worker(vdpasim->worker);
744
745         for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
746                 vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov);
747                 vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov);
748         }
749
750         vdpasim->dev_attr.free(vdpasim);
751
752         for (i = 0; i < vdpasim->dev_attr.nas; i++)
753                 vhost_iotlb_reset(&vdpasim->iommu[i]);
754         kfree(vdpasim->iommu);
755         kfree(vdpasim->iommu_pt);
756         kfree(vdpasim->vqs);
757         kfree(vdpasim->config);
758 }
759
760 static const struct vdpa_config_ops vdpasim_config_ops = {
761         .set_vq_address         = vdpasim_set_vq_address,
762         .set_vq_num             = vdpasim_set_vq_num,
763         .kick_vq                = vdpasim_kick_vq,
764         .set_vq_cb              = vdpasim_set_vq_cb,
765         .set_vq_ready           = vdpasim_set_vq_ready,
766         .get_vq_ready           = vdpasim_get_vq_ready,
767         .set_vq_state           = vdpasim_set_vq_state,
768         .get_vendor_vq_stats    = vdpasim_get_vq_stats,
769         .get_vq_state           = vdpasim_get_vq_state,
770         .get_vq_align           = vdpasim_get_vq_align,
771         .get_vq_group           = vdpasim_get_vq_group,
772         .get_device_features    = vdpasim_get_device_features,
773         .get_backend_features   = vdpasim_get_backend_features,
774         .set_driver_features    = vdpasim_set_driver_features,
775         .get_driver_features    = vdpasim_get_driver_features,
776         .set_config_cb          = vdpasim_set_config_cb,
777         .get_vq_num_max         = vdpasim_get_vq_num_max,
778         .get_device_id          = vdpasim_get_device_id,
779         .get_vendor_id          = vdpasim_get_vendor_id,
780         .get_status             = vdpasim_get_status,
781         .set_status             = vdpasim_set_status,
782         .reset                  = vdpasim_reset,
783         .compat_reset           = vdpasim_compat_reset,
784         .suspend                = vdpasim_suspend,
785         .resume                 = vdpasim_resume,
786         .get_config_size        = vdpasim_get_config_size,
787         .get_config             = vdpasim_get_config,
788         .set_config             = vdpasim_set_config,
789         .get_generation         = vdpasim_get_generation,
790         .get_iova_range         = vdpasim_get_iova_range,
791         .set_group_asid         = vdpasim_set_group_asid,
792         .dma_map                = vdpasim_dma_map,
793         .dma_unmap              = vdpasim_dma_unmap,
794         .reset_map              = vdpasim_reset_map,
795         .bind_mm                = vdpasim_bind_mm,
796         .unbind_mm              = vdpasim_unbind_mm,
797         .free                   = vdpasim_free,
798 };
799
800 static const struct vdpa_config_ops vdpasim_batch_config_ops = {
801         .set_vq_address         = vdpasim_set_vq_address,
802         .set_vq_num             = vdpasim_set_vq_num,
803         .kick_vq                = vdpasim_kick_vq,
804         .set_vq_cb              = vdpasim_set_vq_cb,
805         .set_vq_ready           = vdpasim_set_vq_ready,
806         .get_vq_ready           = vdpasim_get_vq_ready,
807         .set_vq_state           = vdpasim_set_vq_state,
808         .get_vendor_vq_stats    = vdpasim_get_vq_stats,
809         .get_vq_state           = vdpasim_get_vq_state,
810         .get_vq_align           = vdpasim_get_vq_align,
811         .get_vq_group           = vdpasim_get_vq_group,
812         .get_device_features    = vdpasim_get_device_features,
813         .get_backend_features   = vdpasim_get_backend_features,
814         .set_driver_features    = vdpasim_set_driver_features,
815         .get_driver_features    = vdpasim_get_driver_features,
816         .set_config_cb          = vdpasim_set_config_cb,
817         .get_vq_num_max         = vdpasim_get_vq_num_max,
818         .get_device_id          = vdpasim_get_device_id,
819         .get_vendor_id          = vdpasim_get_vendor_id,
820         .get_status             = vdpasim_get_status,
821         .set_status             = vdpasim_set_status,
822         .reset                  = vdpasim_reset,
823         .compat_reset           = vdpasim_compat_reset,
824         .suspend                = vdpasim_suspend,
825         .resume                 = vdpasim_resume,
826         .get_config_size        = vdpasim_get_config_size,
827         .get_config             = vdpasim_get_config,
828         .set_config             = vdpasim_set_config,
829         .get_generation         = vdpasim_get_generation,
830         .get_iova_range         = vdpasim_get_iova_range,
831         .set_group_asid         = vdpasim_set_group_asid,
832         .set_map                = vdpasim_set_map,
833         .reset_map              = vdpasim_reset_map,
834         .bind_mm                = vdpasim_bind_mm,
835         .unbind_mm              = vdpasim_unbind_mm,
836         .free                   = vdpasim_free,
837 };
838
839 MODULE_VERSION(DRV_VERSION);
840 MODULE_LICENSE(DRV_LICENSE);
841 MODULE_AUTHOR(DRV_AUTHOR);
842 MODULE_DESCRIPTION(DRV_DESC);