Mention branches and keyring.
[releases.git] / mlx5 / main.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
4  */
5
6 #include <linux/device.h>
7 #include <linux/eventfd.h>
8 #include <linux/file.h>
9 #include <linux/interrupt.h>
10 #include <linux/iommu.h>
11 #include <linux/module.h>
12 #include <linux/mutex.h>
13 #include <linux/notifier.h>
14 #include <linux/pci.h>
15 #include <linux/pm_runtime.h>
16 #include <linux/types.h>
17 #include <linux/uaccess.h>
18 #include <linux/vfio.h>
19 #include <linux/sched/mm.h>
20 #include <linux/anon_inodes.h>
21
22 #include "cmd.h"
23
24 /* Device specification max LOAD size */
25 #define MAX_LOAD_SIZE (BIT_ULL(__mlx5_bit_sz(load_vhca_state_in, size)) - 1)
26
27 #define MAX_CHUNK_SIZE SZ_8M
28
29 static struct mlx5vf_pci_core_device *mlx5vf_drvdata(struct pci_dev *pdev)
30 {
31         struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
32
33         return container_of(core_device, struct mlx5vf_pci_core_device,
34                             core_device);
35 }
36
37 struct page *
38 mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,
39                           unsigned long offset)
40 {
41         unsigned long cur_offset = 0;
42         struct scatterlist *sg;
43         unsigned int i;
44
45         /* All accesses are sequential */
46         if (offset < buf->last_offset || !buf->last_offset_sg) {
47                 buf->last_offset = 0;
48                 buf->last_offset_sg = buf->table.sgt.sgl;
49                 buf->sg_last_entry = 0;
50         }
51
52         cur_offset = buf->last_offset;
53
54         for_each_sg(buf->last_offset_sg, sg,
55                         buf->table.sgt.orig_nents - buf->sg_last_entry, i) {
56                 if (offset < sg->length + cur_offset) {
57                         buf->last_offset_sg = sg;
58                         buf->sg_last_entry += i;
59                         buf->last_offset = cur_offset;
60                         return nth_page(sg_page(sg),
61                                         (offset - cur_offset) / PAGE_SIZE);
62                 }
63                 cur_offset += sg->length;
64         }
65         return NULL;
66 }
67
68 int mlx5vf_add_migration_pages(struct mlx5_vhca_data_buffer *buf,
69                                unsigned int npages)
70 {
71         unsigned int to_alloc = npages;
72         struct page **page_list;
73         unsigned long filled;
74         unsigned int to_fill;
75         int ret;
76
77         to_fill = min_t(unsigned int, npages, PAGE_SIZE / sizeof(*page_list));
78         page_list = kvzalloc(to_fill * sizeof(*page_list), GFP_KERNEL_ACCOUNT);
79         if (!page_list)
80                 return -ENOMEM;
81
82         do {
83                 filled = alloc_pages_bulk_array(GFP_KERNEL_ACCOUNT, to_fill,
84                                                 page_list);
85                 if (!filled) {
86                         ret = -ENOMEM;
87                         goto err;
88                 }
89                 to_alloc -= filled;
90                 ret = sg_alloc_append_table_from_pages(
91                         &buf->table, page_list, filled, 0,
92                         filled << PAGE_SHIFT, UINT_MAX, SG_MAX_SINGLE_ALLOC,
93                         GFP_KERNEL_ACCOUNT);
94
95                 if (ret)
96                         goto err;
97                 buf->allocated_length += filled * PAGE_SIZE;
98                 /* clean input for another bulk allocation */
99                 memset(page_list, 0, filled * sizeof(*page_list));
100                 to_fill = min_t(unsigned int, to_alloc,
101                                 PAGE_SIZE / sizeof(*page_list));
102         } while (to_alloc > 0);
103
104         kvfree(page_list);
105         return 0;
106
107 err:
108         kvfree(page_list);
109         return ret;
110 }
111
112 static void mlx5vf_disable_fd(struct mlx5_vf_migration_file *migf)
113 {
114         mutex_lock(&migf->lock);
115         migf->state = MLX5_MIGF_STATE_ERROR;
116         migf->filp->f_pos = 0;
117         mutex_unlock(&migf->lock);
118 }
119
120 static int mlx5vf_release_file(struct inode *inode, struct file *filp)
121 {
122         struct mlx5_vf_migration_file *migf = filp->private_data;
123
124         mlx5vf_disable_fd(migf);
125         mutex_destroy(&migf->lock);
126         kfree(migf);
127         return 0;
128 }
129
130 static struct mlx5_vhca_data_buffer *
131 mlx5vf_get_data_buff_from_pos(struct mlx5_vf_migration_file *migf, loff_t pos,
132                               bool *end_of_data)
133 {
134         struct mlx5_vhca_data_buffer *buf;
135         bool found = false;
136
137         *end_of_data = false;
138         spin_lock_irq(&migf->list_lock);
139         if (list_empty(&migf->buf_list)) {
140                 *end_of_data = true;
141                 goto end;
142         }
143
144         buf = list_first_entry(&migf->buf_list, struct mlx5_vhca_data_buffer,
145                                buf_elm);
146         if (pos >= buf->start_pos &&
147             pos < buf->start_pos + buf->length) {
148                 found = true;
149                 goto end;
150         }
151
152         /*
153          * As we use a stream based FD we may expect having the data always
154          * on first chunk
155          */
156         migf->state = MLX5_MIGF_STATE_ERROR;
157
158 end:
159         spin_unlock_irq(&migf->list_lock);
160         return found ? buf : NULL;
161 }
162
163 static void mlx5vf_buf_read_done(struct mlx5_vhca_data_buffer *vhca_buf)
164 {
165         struct mlx5_vf_migration_file *migf = vhca_buf->migf;
166
167         if (vhca_buf->stop_copy_chunk_num) {
168                 bool is_header = vhca_buf->dma_dir == DMA_NONE;
169                 u8 chunk_num = vhca_buf->stop_copy_chunk_num;
170                 size_t next_required_umem_size = 0;
171
172                 if (is_header)
173                         migf->buf_header[chunk_num - 1] = vhca_buf;
174                 else
175                         migf->buf[chunk_num - 1] = vhca_buf;
176
177                 spin_lock_irq(&migf->list_lock);
178                 list_del_init(&vhca_buf->buf_elm);
179                 if (!is_header) {
180                         next_required_umem_size =
181                                 migf->next_required_umem_size;
182                         migf->next_required_umem_size = 0;
183                         migf->num_ready_chunks--;
184                 }
185                 spin_unlock_irq(&migf->list_lock);
186                 if (next_required_umem_size)
187                         mlx5vf_mig_file_set_save_work(migf, chunk_num,
188                                                       next_required_umem_size);
189                 return;
190         }
191
192         spin_lock_irq(&migf->list_lock);
193         list_del_init(&vhca_buf->buf_elm);
194         list_add_tail(&vhca_buf->buf_elm, &vhca_buf->migf->avail_list);
195         spin_unlock_irq(&migf->list_lock);
196 }
197
198 static ssize_t mlx5vf_buf_read(struct mlx5_vhca_data_buffer *vhca_buf,
199                                char __user **buf, size_t *len, loff_t *pos)
200 {
201         unsigned long offset;
202         ssize_t done = 0;
203         size_t copy_len;
204
205         copy_len = min_t(size_t,
206                          vhca_buf->start_pos + vhca_buf->length - *pos, *len);
207         while (copy_len) {
208                 size_t page_offset;
209                 struct page *page;
210                 size_t page_len;
211                 u8 *from_buff;
212                 int ret;
213
214                 offset = *pos - vhca_buf->start_pos;
215                 page_offset = offset % PAGE_SIZE;
216                 offset -= page_offset;
217                 page = mlx5vf_get_migration_page(vhca_buf, offset);
218                 if (!page)
219                         return -EINVAL;
220                 page_len = min_t(size_t, copy_len, PAGE_SIZE - page_offset);
221                 from_buff = kmap_local_page(page);
222                 ret = copy_to_user(*buf, from_buff + page_offset, page_len);
223                 kunmap_local(from_buff);
224                 if (ret)
225                         return -EFAULT;
226                 *pos += page_len;
227                 *len -= page_len;
228                 *buf += page_len;
229                 done += page_len;
230                 copy_len -= page_len;
231         }
232
233         if (*pos >= vhca_buf->start_pos + vhca_buf->length)
234                 mlx5vf_buf_read_done(vhca_buf);
235
236         return done;
237 }
238
239 static ssize_t mlx5vf_save_read(struct file *filp, char __user *buf, size_t len,
240                                loff_t *pos)
241 {
242         struct mlx5_vf_migration_file *migf = filp->private_data;
243         struct mlx5_vhca_data_buffer *vhca_buf;
244         bool first_loop_call = true;
245         bool end_of_data;
246         ssize_t done = 0;
247
248         if (pos)
249                 return -ESPIPE;
250         pos = &filp->f_pos;
251
252         if (!(filp->f_flags & O_NONBLOCK)) {
253                 if (wait_event_interruptible(migf->poll_wait,
254                                 !list_empty(&migf->buf_list) ||
255                                 migf->state == MLX5_MIGF_STATE_ERROR ||
256                                 migf->state == MLX5_MIGF_STATE_PRE_COPY_ERROR ||
257                                 migf->state == MLX5_MIGF_STATE_PRE_COPY ||
258                                 migf->state == MLX5_MIGF_STATE_COMPLETE))
259                         return -ERESTARTSYS;
260         }
261
262         mutex_lock(&migf->lock);
263         if (migf->state == MLX5_MIGF_STATE_ERROR) {
264                 done = -ENODEV;
265                 goto out_unlock;
266         }
267
268         while (len) {
269                 ssize_t count;
270
271                 vhca_buf = mlx5vf_get_data_buff_from_pos(migf, *pos,
272                                                          &end_of_data);
273                 if (first_loop_call) {
274                         first_loop_call = false;
275                         /* Temporary end of file as part of PRE_COPY */
276                         if (end_of_data && (migf->state == MLX5_MIGF_STATE_PRE_COPY ||
277                                 migf->state == MLX5_MIGF_STATE_PRE_COPY_ERROR)) {
278                                 done = -ENOMSG;
279                                 goto out_unlock;
280                         }
281
282                         if (end_of_data && migf->state != MLX5_MIGF_STATE_COMPLETE) {
283                                 if (filp->f_flags & O_NONBLOCK) {
284                                         done = -EAGAIN;
285                                         goto out_unlock;
286                                 }
287                         }
288                 }
289
290                 if (end_of_data)
291                         goto out_unlock;
292
293                 if (!vhca_buf) {
294                         done = -EINVAL;
295                         goto out_unlock;
296                 }
297
298                 count = mlx5vf_buf_read(vhca_buf, &buf, &len, pos);
299                 if (count < 0) {
300                         done = count;
301                         goto out_unlock;
302                 }
303                 done += count;
304         }
305
306 out_unlock:
307         mutex_unlock(&migf->lock);
308         return done;
309 }
310
311 static __poll_t mlx5vf_save_poll(struct file *filp,
312                                  struct poll_table_struct *wait)
313 {
314         struct mlx5_vf_migration_file *migf = filp->private_data;
315         __poll_t pollflags = 0;
316
317         poll_wait(filp, &migf->poll_wait, wait);
318
319         mutex_lock(&migf->lock);
320         if (migf->state == MLX5_MIGF_STATE_ERROR)
321                 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
322         else if (!list_empty(&migf->buf_list) ||
323                  migf->state == MLX5_MIGF_STATE_COMPLETE)
324                 pollflags = EPOLLIN | EPOLLRDNORM;
325         mutex_unlock(&migf->lock);
326
327         return pollflags;
328 }
329
330 /*
331  * FD is exposed and user can use it after receiving an error.
332  * Mark migf in error, and wake the user.
333  */
334 static void mlx5vf_mark_err(struct mlx5_vf_migration_file *migf)
335 {
336         migf->state = MLX5_MIGF_STATE_ERROR;
337         wake_up_interruptible(&migf->poll_wait);
338 }
339
340 void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,
341                                    u8 chunk_num, size_t next_required_umem_size)
342 {
343         migf->save_data[chunk_num - 1].next_required_umem_size =
344                         next_required_umem_size;
345         migf->save_data[chunk_num - 1].migf = migf;
346         get_file(migf->filp);
347         queue_work(migf->mvdev->cb_wq,
348                    &migf->save_data[chunk_num - 1].work);
349 }
350
351 static struct mlx5_vhca_data_buffer *
352 mlx5vf_mig_file_get_stop_copy_buf(struct mlx5_vf_migration_file *migf,
353                                   u8 index, size_t required_length)
354 {
355         struct mlx5_vhca_data_buffer *buf = migf->buf[index];
356         u8 chunk_num;
357
358         WARN_ON(!buf);
359         chunk_num = buf->stop_copy_chunk_num;
360         buf->migf->buf[index] = NULL;
361         /* Checking whether the pre-allocated buffer can fit */
362         if (buf->allocated_length >= required_length)
363                 return buf;
364
365         mlx5vf_put_data_buffer(buf);
366         buf = mlx5vf_get_data_buffer(buf->migf, required_length,
367                                      DMA_FROM_DEVICE);
368         if (IS_ERR(buf))
369                 return buf;
370
371         buf->stop_copy_chunk_num = chunk_num;
372         return buf;
373 }
374
375 static void mlx5vf_mig_file_save_work(struct work_struct *_work)
376 {
377         struct mlx5vf_save_work_data *save_data = container_of(_work,
378                 struct mlx5vf_save_work_data, work);
379         struct mlx5_vf_migration_file *migf = save_data->migf;
380         struct mlx5vf_pci_core_device *mvdev = migf->mvdev;
381         struct mlx5_vhca_data_buffer *buf;
382
383         mutex_lock(&mvdev->state_mutex);
384         if (migf->state == MLX5_MIGF_STATE_ERROR)
385                 goto end;
386
387         buf = mlx5vf_mig_file_get_stop_copy_buf(migf,
388                                 save_data->chunk_num - 1,
389                                 save_data->next_required_umem_size);
390         if (IS_ERR(buf))
391                 goto err;
392
393         if (mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, true, false))
394                 goto err_save;
395
396         goto end;
397
398 err_save:
399         mlx5vf_put_data_buffer(buf);
400 err:
401         mlx5vf_mark_err(migf);
402 end:
403         mlx5vf_state_mutex_unlock(mvdev);
404         fput(migf->filp);
405 }
406
407 static int mlx5vf_add_stop_copy_header(struct mlx5_vf_migration_file *migf,
408                                        bool track)
409 {
410         size_t size = sizeof(struct mlx5_vf_migration_header) +
411                 sizeof(struct mlx5_vf_migration_tag_stop_copy_data);
412         struct mlx5_vf_migration_tag_stop_copy_data data = {};
413         struct mlx5_vhca_data_buffer *header_buf = NULL;
414         struct mlx5_vf_migration_header header = {};
415         unsigned long flags;
416         struct page *page;
417         u8 *to_buff;
418         int ret;
419
420         header_buf = mlx5vf_get_data_buffer(migf, size, DMA_NONE);
421         if (IS_ERR(header_buf))
422                 return PTR_ERR(header_buf);
423
424         header.record_size = cpu_to_le64(sizeof(data));
425         header.flags = cpu_to_le32(MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL);
426         header.tag = cpu_to_le32(MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE);
427         page = mlx5vf_get_migration_page(header_buf, 0);
428         if (!page) {
429                 ret = -EINVAL;
430                 goto err;
431         }
432         to_buff = kmap_local_page(page);
433         memcpy(to_buff, &header, sizeof(header));
434         header_buf->length = sizeof(header);
435         data.stop_copy_size = cpu_to_le64(migf->buf[0]->allocated_length);
436         memcpy(to_buff + sizeof(header), &data, sizeof(data));
437         header_buf->length += sizeof(data);
438         kunmap_local(to_buff);
439         header_buf->start_pos = header_buf->migf->max_pos;
440         migf->max_pos += header_buf->length;
441         spin_lock_irqsave(&migf->list_lock, flags);
442         list_add_tail(&header_buf->buf_elm, &migf->buf_list);
443         spin_unlock_irqrestore(&migf->list_lock, flags);
444         if (track)
445                 migf->pre_copy_initial_bytes = size;
446         return 0;
447 err:
448         mlx5vf_put_data_buffer(header_buf);
449         return ret;
450 }
451
452 static int mlx5vf_prep_stop_copy(struct mlx5vf_pci_core_device *mvdev,
453                                  struct mlx5_vf_migration_file *migf,
454                                  size_t state_size, u64 full_size,
455                                  bool track)
456 {
457         struct mlx5_vhca_data_buffer *buf;
458         size_t inc_state_size;
459         int num_chunks;
460         int ret;
461         int i;
462
463         if (mvdev->chunk_mode) {
464                 size_t chunk_size = min_t(size_t, MAX_CHUNK_SIZE, full_size);
465
466                 /* from firmware perspective at least 'state_size' buffer should be set */
467                 inc_state_size = max(state_size, chunk_size);
468         } else {
469                 if (track) {
470                         /* let's be ready for stop_copy size that might grow by 10 percents */
471                         if (check_add_overflow(state_size, state_size / 10, &inc_state_size))
472                                 inc_state_size = state_size;
473                 } else {
474                         inc_state_size = state_size;
475                 }
476         }
477
478         /* let's not overflow the device specification max SAVE size */
479         inc_state_size = min_t(size_t, inc_state_size,
480                 (BIT_ULL(__mlx5_bit_sz(save_vhca_state_in, size)) - PAGE_SIZE));
481
482         num_chunks = mvdev->chunk_mode ? MAX_NUM_CHUNKS : 1;
483         for (i = 0; i < num_chunks; i++) {
484                 buf = mlx5vf_get_data_buffer(migf, inc_state_size, DMA_FROM_DEVICE);
485                 if (IS_ERR(buf)) {
486                         ret = PTR_ERR(buf);
487                         goto err;
488                 }
489
490                 migf->buf[i] = buf;
491                 buf = mlx5vf_get_data_buffer(migf,
492                                 sizeof(struct mlx5_vf_migration_header), DMA_NONE);
493                 if (IS_ERR(buf)) {
494                         ret = PTR_ERR(buf);
495                         goto err;
496                 }
497                 migf->buf_header[i] = buf;
498                 if (mvdev->chunk_mode) {
499                         migf->buf[i]->stop_copy_chunk_num = i + 1;
500                         migf->buf_header[i]->stop_copy_chunk_num = i + 1;
501                         INIT_WORK(&migf->save_data[i].work,
502                                   mlx5vf_mig_file_save_work);
503                         migf->save_data[i].chunk_num = i + 1;
504                 }
505         }
506
507         ret = mlx5vf_add_stop_copy_header(migf, track);
508         if (ret)
509                 goto err;
510         return 0;
511
512 err:
513         for (i = 0; i < num_chunks; i++) {
514                 if (migf->buf[i]) {
515                         mlx5vf_put_data_buffer(migf->buf[i]);
516                         migf->buf[i] = NULL;
517                 }
518                 if (migf->buf_header[i]) {
519                         mlx5vf_put_data_buffer(migf->buf_header[i]);
520                         migf->buf_header[i] = NULL;
521                 }
522         }
523
524         return ret;
525 }
526
527 static long mlx5vf_precopy_ioctl(struct file *filp, unsigned int cmd,
528                                  unsigned long arg)
529 {
530         struct mlx5_vf_migration_file *migf = filp->private_data;
531         struct mlx5vf_pci_core_device *mvdev = migf->mvdev;
532         struct mlx5_vhca_data_buffer *buf;
533         struct vfio_precopy_info info = {};
534         loff_t *pos = &filp->f_pos;
535         unsigned long minsz;
536         size_t inc_length = 0;
537         bool end_of_data = false;
538         int ret;
539
540         if (cmd != VFIO_MIG_GET_PRECOPY_INFO)
541                 return -ENOTTY;
542
543         minsz = offsetofend(struct vfio_precopy_info, dirty_bytes);
544
545         if (copy_from_user(&info, (void __user *)arg, minsz))
546                 return -EFAULT;
547
548         if (info.argsz < minsz)
549                 return -EINVAL;
550
551         mutex_lock(&mvdev->state_mutex);
552         if (mvdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY &&
553             mvdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY_P2P) {
554                 ret = -EINVAL;
555                 goto err_state_unlock;
556         }
557
558         /*
559          * We can't issue a SAVE command when the device is suspended, so as
560          * part of VFIO_DEVICE_STATE_PRE_COPY_P2P no reason to query for extra
561          * bytes that can't be read.
562          */
563         if (mvdev->mig_state == VFIO_DEVICE_STATE_PRE_COPY) {
564                 /*
565                  * Once the query returns it's guaranteed that there is no
566                  * active SAVE command.
567                  * As so, the other code below is safe with the proper locks.
568                  */
569                 ret = mlx5vf_cmd_query_vhca_migration_state(mvdev, &inc_length,
570                                                             NULL, MLX5VF_QUERY_INC);
571                 if (ret)
572                         goto err_state_unlock;
573         }
574
575         mutex_lock(&migf->lock);
576         if (migf->state == MLX5_MIGF_STATE_ERROR) {
577                 ret = -ENODEV;
578                 goto err_migf_unlock;
579         }
580
581         if (migf->pre_copy_initial_bytes > *pos) {
582                 info.initial_bytes = migf->pre_copy_initial_bytes - *pos;
583         } else {
584                 info.dirty_bytes = migf->max_pos - *pos;
585                 if (!info.dirty_bytes)
586                         end_of_data = true;
587                 info.dirty_bytes += inc_length;
588         }
589
590         if (!end_of_data || !inc_length) {
591                 mutex_unlock(&migf->lock);
592                 goto done;
593         }
594
595         mutex_unlock(&migf->lock);
596         /*
597          * We finished transferring the current state and the device has a
598          * dirty state, save a new state to be ready for.
599          */
600         buf = mlx5vf_get_data_buffer(migf, inc_length, DMA_FROM_DEVICE);
601         if (IS_ERR(buf)) {
602                 ret = PTR_ERR(buf);
603                 mlx5vf_mark_err(migf);
604                 goto err_state_unlock;
605         }
606
607         ret = mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, true, true);
608         if (ret) {
609                 mlx5vf_mark_err(migf);
610                 mlx5vf_put_data_buffer(buf);
611                 goto err_state_unlock;
612         }
613
614 done:
615         mlx5vf_state_mutex_unlock(mvdev);
616         if (copy_to_user((void __user *)arg, &info, minsz))
617                 return -EFAULT;
618         return 0;
619
620 err_migf_unlock:
621         mutex_unlock(&migf->lock);
622 err_state_unlock:
623         mlx5vf_state_mutex_unlock(mvdev);
624         return ret;
625 }
626
627 static const struct file_operations mlx5vf_save_fops = {
628         .owner = THIS_MODULE,
629         .read = mlx5vf_save_read,
630         .poll = mlx5vf_save_poll,
631         .unlocked_ioctl = mlx5vf_precopy_ioctl,
632         .compat_ioctl = compat_ptr_ioctl,
633         .release = mlx5vf_release_file,
634         .llseek = no_llseek,
635 };
636
637 static int mlx5vf_pci_save_device_inc_data(struct mlx5vf_pci_core_device *mvdev)
638 {
639         struct mlx5_vf_migration_file *migf = mvdev->saving_migf;
640         struct mlx5_vhca_data_buffer *buf;
641         size_t length;
642         int ret;
643
644         if (migf->state == MLX5_MIGF_STATE_ERROR)
645                 return -ENODEV;
646
647         ret = mlx5vf_cmd_query_vhca_migration_state(mvdev, &length, NULL,
648                                 MLX5VF_QUERY_INC | MLX5VF_QUERY_FINAL);
649         if (ret)
650                 goto err;
651
652         buf = mlx5vf_mig_file_get_stop_copy_buf(migf, 0, length);
653         if (IS_ERR(buf)) {
654                 ret = PTR_ERR(buf);
655                 goto err;
656         }
657
658         ret = mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, true, false);
659         if (ret)
660                 goto err_save;
661
662         return 0;
663
664 err_save:
665         mlx5vf_put_data_buffer(buf);
666 err:
667         mlx5vf_mark_err(migf);
668         return ret;
669 }
670
671 static struct mlx5_vf_migration_file *
672 mlx5vf_pci_save_device_data(struct mlx5vf_pci_core_device *mvdev, bool track)
673 {
674         struct mlx5_vf_migration_file *migf;
675         struct mlx5_vhca_data_buffer *buf;
676         size_t length;
677         u64 full_size;
678         int ret;
679
680         migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);
681         if (!migf)
682                 return ERR_PTR(-ENOMEM);
683
684         migf->filp = anon_inode_getfile("mlx5vf_mig", &mlx5vf_save_fops, migf,
685                                         O_RDONLY);
686         if (IS_ERR(migf->filp)) {
687                 ret = PTR_ERR(migf->filp);
688                 goto end;
689         }
690
691         migf->mvdev = mvdev;
692         ret = mlx5vf_cmd_alloc_pd(migf);
693         if (ret)
694                 goto out_free;
695
696         stream_open(migf->filp->f_inode, migf->filp);
697         mutex_init(&migf->lock);
698         init_waitqueue_head(&migf->poll_wait);
699         init_completion(&migf->save_comp);
700         /*
701          * save_comp is being used as a binary semaphore built from
702          * a completion. A normal mutex cannot be used because the lock is
703          * passed between kernel threads and lockdep can't model this.
704          */
705         complete(&migf->save_comp);
706         mlx5_cmd_init_async_ctx(mvdev->mdev, &migf->async_ctx);
707         INIT_WORK(&migf->async_data.work, mlx5vf_mig_file_cleanup_cb);
708         INIT_LIST_HEAD(&migf->buf_list);
709         INIT_LIST_HEAD(&migf->avail_list);
710         spin_lock_init(&migf->list_lock);
711         ret = mlx5vf_cmd_query_vhca_migration_state(mvdev, &length, &full_size, 0);
712         if (ret)
713                 goto out_pd;
714
715         ret = mlx5vf_prep_stop_copy(mvdev, migf, length, full_size, track);
716         if (ret)
717                 goto out_pd;
718
719         if (track) {
720                 /* leave the allocated buffer ready for the stop-copy phase */
721                 buf = mlx5vf_alloc_data_buffer(migf,
722                         migf->buf[0]->allocated_length, DMA_FROM_DEVICE);
723                 if (IS_ERR(buf)) {
724                         ret = PTR_ERR(buf);
725                         goto out_pd;
726                 }
727         } else {
728                 buf = migf->buf[0];
729                 migf->buf[0] = NULL;
730         }
731
732         ret = mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, false, track);
733         if (ret)
734                 goto out_save;
735         return migf;
736 out_save:
737         mlx5vf_free_data_buffer(buf);
738 out_pd:
739         mlx5fv_cmd_clean_migf_resources(migf);
740 out_free:
741         fput(migf->filp);
742 end:
743         kfree(migf);
744         return ERR_PTR(ret);
745 }
746
747 static int
748 mlx5vf_append_page_to_mig_buf(struct mlx5_vhca_data_buffer *vhca_buf,
749                               const char __user **buf, size_t *len,
750                               loff_t *pos, ssize_t *done)
751 {
752         unsigned long offset;
753         size_t page_offset;
754         struct page *page;
755         size_t page_len;
756         u8 *to_buff;
757         int ret;
758
759         offset = *pos - vhca_buf->start_pos;
760         page_offset = offset % PAGE_SIZE;
761
762         page = mlx5vf_get_migration_page(vhca_buf, offset - page_offset);
763         if (!page)
764                 return -EINVAL;
765         page_len = min_t(size_t, *len, PAGE_SIZE - page_offset);
766         to_buff = kmap_local_page(page);
767         ret = copy_from_user(to_buff + page_offset, *buf, page_len);
768         kunmap_local(to_buff);
769         if (ret)
770                 return -EFAULT;
771
772         *pos += page_len;
773         *done += page_len;
774         *buf += page_len;
775         *len -= page_len;
776         vhca_buf->length += page_len;
777         return 0;
778 }
779
780 static int
781 mlx5vf_resume_read_image_no_header(struct mlx5_vhca_data_buffer *vhca_buf,
782                                    loff_t requested_length,
783                                    const char __user **buf, size_t *len,
784                                    loff_t *pos, ssize_t *done)
785 {
786         int ret;
787
788         if (requested_length > MAX_LOAD_SIZE)
789                 return -ENOMEM;
790
791         if (vhca_buf->allocated_length < requested_length) {
792                 ret = mlx5vf_add_migration_pages(
793                         vhca_buf,
794                         DIV_ROUND_UP(requested_length - vhca_buf->allocated_length,
795                                      PAGE_SIZE));
796                 if (ret)
797                         return ret;
798         }
799
800         while (*len) {
801                 ret = mlx5vf_append_page_to_mig_buf(vhca_buf, buf, len, pos,
802                                                     done);
803                 if (ret)
804                         return ret;
805         }
806
807         return 0;
808 }
809
810 static ssize_t
811 mlx5vf_resume_read_image(struct mlx5_vf_migration_file *migf,
812                          struct mlx5_vhca_data_buffer *vhca_buf,
813                          size_t image_size, const char __user **buf,
814                          size_t *len, loff_t *pos, ssize_t *done,
815                          bool *has_work)
816 {
817         size_t copy_len, to_copy;
818         int ret;
819
820         to_copy = min_t(size_t, *len, image_size - vhca_buf->length);
821         copy_len = to_copy;
822         while (to_copy) {
823                 ret = mlx5vf_append_page_to_mig_buf(vhca_buf, buf, &to_copy, pos,
824                                                     done);
825                 if (ret)
826                         return ret;
827         }
828
829         *len -= copy_len;
830         if (vhca_buf->length == image_size) {
831                 migf->load_state = MLX5_VF_LOAD_STATE_LOAD_IMAGE;
832                 migf->max_pos += image_size;
833                 *has_work = true;
834         }
835
836         return 0;
837 }
838
839 static int
840 mlx5vf_resume_read_header_data(struct mlx5_vf_migration_file *migf,
841                                struct mlx5_vhca_data_buffer *vhca_buf,
842                                const char __user **buf, size_t *len,
843                                loff_t *pos, ssize_t *done)
844 {
845         size_t copy_len, to_copy;
846         size_t required_data;
847         u8 *to_buff;
848         int ret;
849
850         required_data = migf->record_size - vhca_buf->length;
851         to_copy = min_t(size_t, *len, required_data);
852         copy_len = to_copy;
853         while (to_copy) {
854                 ret = mlx5vf_append_page_to_mig_buf(vhca_buf, buf, &to_copy, pos,
855                                                     done);
856                 if (ret)
857                         return ret;
858         }
859
860         *len -= copy_len;
861         if (vhca_buf->length == migf->record_size) {
862                 switch (migf->record_tag) {
863                 case MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE:
864                 {
865                         struct page *page;
866
867                         page = mlx5vf_get_migration_page(vhca_buf, 0);
868                         if (!page)
869                                 return -EINVAL;
870                         to_buff = kmap_local_page(page);
871                         migf->stop_copy_prep_size = min_t(u64,
872                                 le64_to_cpup((__le64 *)to_buff), MAX_LOAD_SIZE);
873                         kunmap_local(to_buff);
874                         break;
875                 }
876                 default:
877                         /* Optional tag */
878                         break;
879                 }
880
881                 migf->load_state = MLX5_VF_LOAD_STATE_READ_HEADER;
882                 migf->max_pos += migf->record_size;
883                 vhca_buf->length = 0;
884         }
885
886         return 0;
887 }
888
889 static int
890 mlx5vf_resume_read_header(struct mlx5_vf_migration_file *migf,
891                           struct mlx5_vhca_data_buffer *vhca_buf,
892                           const char __user **buf,
893                           size_t *len, loff_t *pos,
894                           ssize_t *done, bool *has_work)
895 {
896         struct page *page;
897         size_t copy_len;
898         u8 *to_buff;
899         int ret;
900
901         copy_len = min_t(size_t, *len,
902                 sizeof(struct mlx5_vf_migration_header) - vhca_buf->length);
903         page = mlx5vf_get_migration_page(vhca_buf, 0);
904         if (!page)
905                 return -EINVAL;
906         to_buff = kmap_local_page(page);
907         ret = copy_from_user(to_buff + vhca_buf->length, *buf, copy_len);
908         if (ret) {
909                 ret = -EFAULT;
910                 goto end;
911         }
912
913         *buf += copy_len;
914         *pos += copy_len;
915         *done += copy_len;
916         *len -= copy_len;
917         vhca_buf->length += copy_len;
918         if (vhca_buf->length == sizeof(struct mlx5_vf_migration_header)) {
919                 u64 record_size;
920                 u32 flags;
921
922                 record_size = le64_to_cpup((__le64 *)to_buff);
923                 if (record_size > MAX_LOAD_SIZE) {
924                         ret = -ENOMEM;
925                         goto end;
926                 }
927
928                 migf->record_size = record_size;
929                 flags = le32_to_cpup((__le32 *)(to_buff +
930                             offsetof(struct mlx5_vf_migration_header, flags)));
931                 migf->record_tag = le32_to_cpup((__le32 *)(to_buff +
932                             offsetof(struct mlx5_vf_migration_header, tag)));
933                 switch (migf->record_tag) {
934                 case MLX5_MIGF_HEADER_TAG_FW_DATA:
935                         migf->load_state = MLX5_VF_LOAD_STATE_PREP_IMAGE;
936                         break;
937                 case MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE:
938                         migf->load_state = MLX5_VF_LOAD_STATE_PREP_HEADER_DATA;
939                         break;
940                 default:
941                         if (!(flags & MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL)) {
942                                 ret = -EOPNOTSUPP;
943                                 goto end;
944                         }
945                         /* We may read and skip this optional record data */
946                         migf->load_state = MLX5_VF_LOAD_STATE_PREP_HEADER_DATA;
947                 }
948
949                 migf->max_pos += vhca_buf->length;
950                 vhca_buf->length = 0;
951                 *has_work = true;
952         }
953 end:
954         kunmap_local(to_buff);
955         return ret;
956 }
957
958 static ssize_t mlx5vf_resume_write(struct file *filp, const char __user *buf,
959                                    size_t len, loff_t *pos)
960 {
961         struct mlx5_vf_migration_file *migf = filp->private_data;
962         struct mlx5_vhca_data_buffer *vhca_buf = migf->buf[0];
963         struct mlx5_vhca_data_buffer *vhca_buf_header = migf->buf_header[0];
964         loff_t requested_length;
965         bool has_work = false;
966         ssize_t done = 0;
967         int ret = 0;
968
969         if (pos)
970                 return -ESPIPE;
971         pos = &filp->f_pos;
972
973         if (*pos < 0 ||
974             check_add_overflow((loff_t)len, *pos, &requested_length))
975                 return -EINVAL;
976
977         mutex_lock(&migf->mvdev->state_mutex);
978         mutex_lock(&migf->lock);
979         if (migf->state == MLX5_MIGF_STATE_ERROR) {
980                 ret = -ENODEV;
981                 goto out_unlock;
982         }
983
984         while (len || has_work) {
985                 has_work = false;
986                 switch (migf->load_state) {
987                 case MLX5_VF_LOAD_STATE_READ_HEADER:
988                         ret = mlx5vf_resume_read_header(migf, vhca_buf_header,
989                                                         &buf, &len, pos,
990                                                         &done, &has_work);
991                         if (ret)
992                                 goto out_unlock;
993                         break;
994                 case MLX5_VF_LOAD_STATE_PREP_HEADER_DATA:
995                         if (vhca_buf_header->allocated_length < migf->record_size) {
996                                 mlx5vf_free_data_buffer(vhca_buf_header);
997
998                                 migf->buf_header[0] = mlx5vf_alloc_data_buffer(migf,
999                                                 migf->record_size, DMA_NONE);
1000                                 if (IS_ERR(migf->buf_header[0])) {
1001                                         ret = PTR_ERR(migf->buf_header[0]);
1002                                         migf->buf_header[0] = NULL;
1003                                         goto out_unlock;
1004                                 }
1005
1006                                 vhca_buf_header = migf->buf_header[0];
1007                         }
1008
1009                         vhca_buf_header->start_pos = migf->max_pos;
1010                         migf->load_state = MLX5_VF_LOAD_STATE_READ_HEADER_DATA;
1011                         break;
1012                 case MLX5_VF_LOAD_STATE_READ_HEADER_DATA:
1013                         ret = mlx5vf_resume_read_header_data(migf, vhca_buf_header,
1014                                                         &buf, &len, pos, &done);
1015                         if (ret)
1016                                 goto out_unlock;
1017                         break;
1018                 case MLX5_VF_LOAD_STATE_PREP_IMAGE:
1019                 {
1020                         u64 size = max(migf->record_size,
1021                                        migf->stop_copy_prep_size);
1022
1023                         if (vhca_buf->allocated_length < size) {
1024                                 mlx5vf_free_data_buffer(vhca_buf);
1025
1026                                 migf->buf[0] = mlx5vf_alloc_data_buffer(migf,
1027                                                         size, DMA_TO_DEVICE);
1028                                 if (IS_ERR(migf->buf[0])) {
1029                                         ret = PTR_ERR(migf->buf[0]);
1030                                         migf->buf[0] = NULL;
1031                                         goto out_unlock;
1032                                 }
1033
1034                                 vhca_buf = migf->buf[0];
1035                         }
1036
1037                         vhca_buf->start_pos = migf->max_pos;
1038                         migf->load_state = MLX5_VF_LOAD_STATE_READ_IMAGE;
1039                         break;
1040                 }
1041                 case MLX5_VF_LOAD_STATE_READ_IMAGE_NO_HEADER:
1042                         ret = mlx5vf_resume_read_image_no_header(vhca_buf,
1043                                                 requested_length,
1044                                                 &buf, &len, pos, &done);
1045                         if (ret)
1046                                 goto out_unlock;
1047                         break;
1048                 case MLX5_VF_LOAD_STATE_READ_IMAGE:
1049                         ret = mlx5vf_resume_read_image(migf, vhca_buf,
1050                                                 migf->record_size,
1051                                                 &buf, &len, pos, &done, &has_work);
1052                         if (ret)
1053                                 goto out_unlock;
1054                         break;
1055                 case MLX5_VF_LOAD_STATE_LOAD_IMAGE:
1056                         ret = mlx5vf_cmd_load_vhca_state(migf->mvdev, migf, vhca_buf);
1057                         if (ret)
1058                                 goto out_unlock;
1059                         migf->load_state = MLX5_VF_LOAD_STATE_READ_HEADER;
1060
1061                         /* prep header buf for next image */
1062                         vhca_buf_header->length = 0;
1063                         /* prep data buf for next image */
1064                         vhca_buf->length = 0;
1065
1066                         break;
1067                 default:
1068                         break;
1069                 }
1070         }
1071
1072 out_unlock:
1073         if (ret)
1074                 migf->state = MLX5_MIGF_STATE_ERROR;
1075         mutex_unlock(&migf->lock);
1076         mlx5vf_state_mutex_unlock(migf->mvdev);
1077         return ret ? ret : done;
1078 }
1079
1080 static const struct file_operations mlx5vf_resume_fops = {
1081         .owner = THIS_MODULE,
1082         .write = mlx5vf_resume_write,
1083         .release = mlx5vf_release_file,
1084         .llseek = no_llseek,
1085 };
1086
1087 static struct mlx5_vf_migration_file *
1088 mlx5vf_pci_resume_device_data(struct mlx5vf_pci_core_device *mvdev)
1089 {
1090         struct mlx5_vf_migration_file *migf;
1091         struct mlx5_vhca_data_buffer *buf;
1092         int ret;
1093
1094         migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);
1095         if (!migf)
1096                 return ERR_PTR(-ENOMEM);
1097
1098         migf->filp = anon_inode_getfile("mlx5vf_mig", &mlx5vf_resume_fops, migf,
1099                                         O_WRONLY);
1100         if (IS_ERR(migf->filp)) {
1101                 ret = PTR_ERR(migf->filp);
1102                 goto end;
1103         }
1104
1105         migf->mvdev = mvdev;
1106         ret = mlx5vf_cmd_alloc_pd(migf);
1107         if (ret)
1108                 goto out_free;
1109
1110         buf = mlx5vf_alloc_data_buffer(migf, 0, DMA_TO_DEVICE);
1111         if (IS_ERR(buf)) {
1112                 ret = PTR_ERR(buf);
1113                 goto out_pd;
1114         }
1115
1116         migf->buf[0] = buf;
1117         if (MLX5VF_PRE_COPY_SUPP(mvdev)) {
1118                 buf = mlx5vf_alloc_data_buffer(migf,
1119                         sizeof(struct mlx5_vf_migration_header), DMA_NONE);
1120                 if (IS_ERR(buf)) {
1121                         ret = PTR_ERR(buf);
1122                         goto out_buf;
1123                 }
1124
1125                 migf->buf_header[0] = buf;
1126                 migf->load_state = MLX5_VF_LOAD_STATE_READ_HEADER;
1127         } else {
1128                 /* Initial state will be to read the image */
1129                 migf->load_state = MLX5_VF_LOAD_STATE_READ_IMAGE_NO_HEADER;
1130         }
1131
1132         stream_open(migf->filp->f_inode, migf->filp);
1133         mutex_init(&migf->lock);
1134         INIT_LIST_HEAD(&migf->buf_list);
1135         INIT_LIST_HEAD(&migf->avail_list);
1136         spin_lock_init(&migf->list_lock);
1137         return migf;
1138 out_buf:
1139         mlx5vf_free_data_buffer(migf->buf[0]);
1140 out_pd:
1141         mlx5vf_cmd_dealloc_pd(migf);
1142 out_free:
1143         fput(migf->filp);
1144 end:
1145         kfree(migf);
1146         return ERR_PTR(ret);
1147 }
1148
1149 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev)
1150 {
1151         if (mvdev->resuming_migf) {
1152                 mlx5vf_disable_fd(mvdev->resuming_migf);
1153                 mlx5fv_cmd_clean_migf_resources(mvdev->resuming_migf);
1154                 fput(mvdev->resuming_migf->filp);
1155                 mvdev->resuming_migf = NULL;
1156         }
1157         if (mvdev->saving_migf) {
1158                 mlx5_cmd_cleanup_async_ctx(&mvdev->saving_migf->async_ctx);
1159                 cancel_work_sync(&mvdev->saving_migf->async_data.work);
1160                 mlx5vf_disable_fd(mvdev->saving_migf);
1161                 wake_up_interruptible(&mvdev->saving_migf->poll_wait);
1162                 mlx5fv_cmd_clean_migf_resources(mvdev->saving_migf);
1163                 fput(mvdev->saving_migf->filp);
1164                 mvdev->saving_migf = NULL;
1165         }
1166 }
1167
1168 static struct file *
1169 mlx5vf_pci_step_device_state_locked(struct mlx5vf_pci_core_device *mvdev,
1170                                     u32 new)
1171 {
1172         u32 cur = mvdev->mig_state;
1173         int ret;
1174
1175         if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_STOP) {
1176                 ret = mlx5vf_cmd_suspend_vhca(mvdev,
1177                         MLX5_SUSPEND_VHCA_IN_OP_MOD_SUSPEND_RESPONDER);
1178                 if (ret)
1179                         return ERR_PTR(ret);
1180                 return NULL;
1181         }
1182
1183         if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING_P2P) {
1184                 ret = mlx5vf_cmd_resume_vhca(mvdev,
1185                         MLX5_RESUME_VHCA_IN_OP_MOD_RESUME_RESPONDER);
1186                 if (ret)
1187                         return ERR_PTR(ret);
1188                 return NULL;
1189         }
1190
1191         if ((cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_RUNNING_P2P) ||
1192             (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_PRE_COPY_P2P)) {
1193                 ret = mlx5vf_cmd_suspend_vhca(mvdev,
1194                         MLX5_SUSPEND_VHCA_IN_OP_MOD_SUSPEND_INITIATOR);
1195                 if (ret)
1196                         return ERR_PTR(ret);
1197                 return NULL;
1198         }
1199
1200         if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_RUNNING) ||
1201             (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_PRE_COPY)) {
1202                 ret = mlx5vf_cmd_resume_vhca(mvdev,
1203                         MLX5_RESUME_VHCA_IN_OP_MOD_RESUME_INITIATOR);
1204                 if (ret)
1205                         return ERR_PTR(ret);
1206                 return NULL;
1207         }
1208
1209         if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) {
1210                 struct mlx5_vf_migration_file *migf;
1211
1212                 migf = mlx5vf_pci_save_device_data(mvdev, false);
1213                 if (IS_ERR(migf))
1214                         return ERR_CAST(migf);
1215                 get_file(migf->filp);
1216                 mvdev->saving_migf = migf;
1217                 return migf->filp;
1218         }
1219
1220         if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP) ||
1221             (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) ||
1222             (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P &&
1223              new == VFIO_DEVICE_STATE_RUNNING_P2P)) {
1224                 mlx5vf_disable_fds(mvdev);
1225                 return NULL;
1226         }
1227
1228         if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) {
1229                 struct mlx5_vf_migration_file *migf;
1230
1231                 migf = mlx5vf_pci_resume_device_data(mvdev);
1232                 if (IS_ERR(migf))
1233                         return ERR_CAST(migf);
1234                 get_file(migf->filp);
1235                 mvdev->resuming_migf = migf;
1236                 return migf->filp;
1237         }
1238
1239         if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) {
1240                 if (!MLX5VF_PRE_COPY_SUPP(mvdev)) {
1241                         ret = mlx5vf_cmd_load_vhca_state(mvdev,
1242                                                          mvdev->resuming_migf,
1243                                                          mvdev->resuming_migf->buf[0]);
1244                         if (ret)
1245                                 return ERR_PTR(ret);
1246                 }
1247                 mlx5vf_disable_fds(mvdev);
1248                 return NULL;
1249         }
1250
1251         if ((cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) ||
1252             (cur == VFIO_DEVICE_STATE_RUNNING_P2P &&
1253              new == VFIO_DEVICE_STATE_PRE_COPY_P2P)) {
1254                 struct mlx5_vf_migration_file *migf;
1255
1256                 migf = mlx5vf_pci_save_device_data(mvdev, true);
1257                 if (IS_ERR(migf))
1258                         return ERR_CAST(migf);
1259                 get_file(migf->filp);
1260                 mvdev->saving_migf = migf;
1261                 return migf->filp;
1262         }
1263
1264         if (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_STOP_COPY) {
1265                 ret = mlx5vf_cmd_suspend_vhca(mvdev,
1266                         MLX5_SUSPEND_VHCA_IN_OP_MOD_SUSPEND_RESPONDER);
1267                 if (ret)
1268                         return ERR_PTR(ret);
1269                 ret = mlx5vf_pci_save_device_inc_data(mvdev);
1270                 return ret ? ERR_PTR(ret) : NULL;
1271         }
1272
1273         /*
1274          * vfio_mig_get_next_state() does not use arcs other than the above
1275          */
1276         WARN_ON(true);
1277         return ERR_PTR(-EINVAL);
1278 }
1279
1280 /*
1281  * This function is called in all state_mutex unlock cases to
1282  * handle a 'deferred_reset' if exists.
1283  */
1284 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev)
1285 {
1286 again:
1287         spin_lock(&mvdev->reset_lock);
1288         if (mvdev->deferred_reset) {
1289                 mvdev->deferred_reset = false;
1290                 spin_unlock(&mvdev->reset_lock);
1291                 mvdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
1292                 mlx5vf_disable_fds(mvdev);
1293                 goto again;
1294         }
1295         mutex_unlock(&mvdev->state_mutex);
1296         spin_unlock(&mvdev->reset_lock);
1297 }
1298
1299 static struct file *
1300 mlx5vf_pci_set_device_state(struct vfio_device *vdev,
1301                             enum vfio_device_mig_state new_state)
1302 {
1303         struct mlx5vf_pci_core_device *mvdev = container_of(
1304                 vdev, struct mlx5vf_pci_core_device, core_device.vdev);
1305         enum vfio_device_mig_state next_state;
1306         struct file *res = NULL;
1307         int ret;
1308
1309         mutex_lock(&mvdev->state_mutex);
1310         while (new_state != mvdev->mig_state) {
1311                 ret = vfio_mig_get_next_state(vdev, mvdev->mig_state,
1312                                               new_state, &next_state);
1313                 if (ret) {
1314                         res = ERR_PTR(ret);
1315                         break;
1316                 }
1317                 res = mlx5vf_pci_step_device_state_locked(mvdev, next_state);
1318                 if (IS_ERR(res))
1319                         break;
1320                 mvdev->mig_state = next_state;
1321                 if (WARN_ON(res && new_state != mvdev->mig_state)) {
1322                         fput(res);
1323                         res = ERR_PTR(-EINVAL);
1324                         break;
1325                 }
1326         }
1327         mlx5vf_state_mutex_unlock(mvdev);
1328         return res;
1329 }
1330
1331 static int mlx5vf_pci_get_data_size(struct vfio_device *vdev,
1332                                     unsigned long *stop_copy_length)
1333 {
1334         struct mlx5vf_pci_core_device *mvdev = container_of(
1335                 vdev, struct mlx5vf_pci_core_device, core_device.vdev);
1336         size_t state_size;
1337         u64 total_size;
1338         int ret;
1339
1340         mutex_lock(&mvdev->state_mutex);
1341         ret = mlx5vf_cmd_query_vhca_migration_state(mvdev, &state_size,
1342                                                     &total_size, 0);
1343         if (!ret)
1344                 *stop_copy_length = total_size;
1345         mlx5vf_state_mutex_unlock(mvdev);
1346         return ret;
1347 }
1348
1349 static int mlx5vf_pci_get_device_state(struct vfio_device *vdev,
1350                                        enum vfio_device_mig_state *curr_state)
1351 {
1352         struct mlx5vf_pci_core_device *mvdev = container_of(
1353                 vdev, struct mlx5vf_pci_core_device, core_device.vdev);
1354
1355         mutex_lock(&mvdev->state_mutex);
1356         *curr_state = mvdev->mig_state;
1357         mlx5vf_state_mutex_unlock(mvdev);
1358         return 0;
1359 }
1360
1361 static void mlx5vf_pci_aer_reset_done(struct pci_dev *pdev)
1362 {
1363         struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev);
1364
1365         if (!mvdev->migrate_cap)
1366                 return;
1367
1368         /*
1369          * As the higher VFIO layers are holding locks across reset and using
1370          * those same locks with the mm_lock we need to prevent ABBA deadlock
1371          * with the state_mutex and mm_lock.
1372          * In case the state_mutex was taken already we defer the cleanup work
1373          * to the unlock flow of the other running context.
1374          */
1375         spin_lock(&mvdev->reset_lock);
1376         mvdev->deferred_reset = true;
1377         if (!mutex_trylock(&mvdev->state_mutex)) {
1378                 spin_unlock(&mvdev->reset_lock);
1379                 return;
1380         }
1381         spin_unlock(&mvdev->reset_lock);
1382         mlx5vf_state_mutex_unlock(mvdev);
1383 }
1384
1385 static int mlx5vf_pci_open_device(struct vfio_device *core_vdev)
1386 {
1387         struct mlx5vf_pci_core_device *mvdev = container_of(
1388                 core_vdev, struct mlx5vf_pci_core_device, core_device.vdev);
1389         struct vfio_pci_core_device *vdev = &mvdev->core_device;
1390         int ret;
1391
1392         ret = vfio_pci_core_enable(vdev);
1393         if (ret)
1394                 return ret;
1395
1396         if (mvdev->migrate_cap)
1397                 mvdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
1398         vfio_pci_core_finish_enable(vdev);
1399         return 0;
1400 }
1401
1402 static void mlx5vf_pci_close_device(struct vfio_device *core_vdev)
1403 {
1404         struct mlx5vf_pci_core_device *mvdev = container_of(
1405                 core_vdev, struct mlx5vf_pci_core_device, core_device.vdev);
1406
1407         mlx5vf_cmd_close_migratable(mvdev);
1408         vfio_pci_core_close_device(core_vdev);
1409 }
1410
1411 static const struct vfio_migration_ops mlx5vf_pci_mig_ops = {
1412         .migration_set_state = mlx5vf_pci_set_device_state,
1413         .migration_get_state = mlx5vf_pci_get_device_state,
1414         .migration_get_data_size = mlx5vf_pci_get_data_size,
1415 };
1416
1417 static const struct vfio_log_ops mlx5vf_pci_log_ops = {
1418         .log_start = mlx5vf_start_page_tracker,
1419         .log_stop = mlx5vf_stop_page_tracker,
1420         .log_read_and_clear = mlx5vf_tracker_read_and_clear,
1421 };
1422
1423 static int mlx5vf_pci_init_dev(struct vfio_device *core_vdev)
1424 {
1425         struct mlx5vf_pci_core_device *mvdev = container_of(core_vdev,
1426                         struct mlx5vf_pci_core_device, core_device.vdev);
1427         int ret;
1428
1429         ret = vfio_pci_core_init_dev(core_vdev);
1430         if (ret)
1431                 return ret;
1432
1433         mlx5vf_cmd_set_migratable(mvdev, &mlx5vf_pci_mig_ops,
1434                                   &mlx5vf_pci_log_ops);
1435
1436         return 0;
1437 }
1438
1439 static void mlx5vf_pci_release_dev(struct vfio_device *core_vdev)
1440 {
1441         struct mlx5vf_pci_core_device *mvdev = container_of(core_vdev,
1442                         struct mlx5vf_pci_core_device, core_device.vdev);
1443
1444         mlx5vf_cmd_remove_migratable(mvdev);
1445         vfio_pci_core_release_dev(core_vdev);
1446 }
1447
1448 static const struct vfio_device_ops mlx5vf_pci_ops = {
1449         .name = "mlx5-vfio-pci",
1450         .init = mlx5vf_pci_init_dev,
1451         .release = mlx5vf_pci_release_dev,
1452         .open_device = mlx5vf_pci_open_device,
1453         .close_device = mlx5vf_pci_close_device,
1454         .ioctl = vfio_pci_core_ioctl,
1455         .device_feature = vfio_pci_core_ioctl_feature,
1456         .read = vfio_pci_core_read,
1457         .write = vfio_pci_core_write,
1458         .mmap = vfio_pci_core_mmap,
1459         .request = vfio_pci_core_request,
1460         .match = vfio_pci_core_match,
1461         .bind_iommufd = vfio_iommufd_physical_bind,
1462         .unbind_iommufd = vfio_iommufd_physical_unbind,
1463         .attach_ioas = vfio_iommufd_physical_attach_ioas,
1464         .detach_ioas = vfio_iommufd_physical_detach_ioas,
1465 };
1466
1467 static int mlx5vf_pci_probe(struct pci_dev *pdev,
1468                             const struct pci_device_id *id)
1469 {
1470         struct mlx5vf_pci_core_device *mvdev;
1471         int ret;
1472
1473         mvdev = vfio_alloc_device(mlx5vf_pci_core_device, core_device.vdev,
1474                                   &pdev->dev, &mlx5vf_pci_ops);
1475         if (IS_ERR(mvdev))
1476                 return PTR_ERR(mvdev);
1477
1478         dev_set_drvdata(&pdev->dev, &mvdev->core_device);
1479         ret = vfio_pci_core_register_device(&mvdev->core_device);
1480         if (ret)
1481                 goto out_put_vdev;
1482         return 0;
1483
1484 out_put_vdev:
1485         vfio_put_device(&mvdev->core_device.vdev);
1486         return ret;
1487 }
1488
1489 static void mlx5vf_pci_remove(struct pci_dev *pdev)
1490 {
1491         struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev);
1492
1493         vfio_pci_core_unregister_device(&mvdev->core_device);
1494         vfio_put_device(&mvdev->core_device.vdev);
1495 }
1496
1497 static const struct pci_device_id mlx5vf_pci_table[] = {
1498         { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0x101e) }, /* ConnectX Family mlx5Gen Virtual Function */
1499         {}
1500 };
1501
1502 MODULE_DEVICE_TABLE(pci, mlx5vf_pci_table);
1503
1504 static const struct pci_error_handlers mlx5vf_err_handlers = {
1505         .reset_done = mlx5vf_pci_aer_reset_done,
1506         .error_detected = vfio_pci_core_aer_err_detected,
1507 };
1508
1509 static struct pci_driver mlx5vf_pci_driver = {
1510         .name = KBUILD_MODNAME,
1511         .id_table = mlx5vf_pci_table,
1512         .probe = mlx5vf_pci_probe,
1513         .remove = mlx5vf_pci_remove,
1514         .err_handler = &mlx5vf_err_handlers,
1515         .driver_managed_dma = true,
1516 };
1517
1518 module_pci_driver(mlx5vf_pci_driver);
1519
1520 MODULE_IMPORT_NS(IOMMUFD);
1521 MODULE_LICENSE("GPL");
1522 MODULE_AUTHOR("Max Gurtovoy <mgurtovoy@nvidia.com>");
1523 MODULE_AUTHOR("Yishai Hadas <yishaih@nvidia.com>");
1524 MODULE_DESCRIPTION(
1525         "MLX5 VFIO PCI - User Level meta-driver for MLX5 device family");