1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */
4 #include <linux/mlx5/driver.h>
7 static int alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid)
9 struct mlx5_core_dev *mdev = dev->mdev;
11 u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
12 u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
15 MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
16 MLX5_SET(alloc_pd_in, in, uid, uid);
18 err = mlx5_cmd_exec_inout(mdev, alloc_pd, in, out);
20 *pdn = MLX5_GET(alloc_pd_out, out, pd);
25 static int dealloc_pd(struct mlx5_vdpa_dev *dev, u32 pdn, u16 uid)
27 u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {};
28 struct mlx5_core_dev *mdev = dev->mdev;
30 MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);
31 MLX5_SET(dealloc_pd_in, in, pd, pdn);
32 MLX5_SET(dealloc_pd_in, in, uid, uid);
33 return mlx5_cmd_exec_in(mdev, dealloc_pd, in);
36 static int get_null_mkey(struct mlx5_vdpa_dev *dev, u32 *null_mkey)
38 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
39 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
40 struct mlx5_core_dev *mdev = dev->mdev;
43 MLX5_SET(query_special_contexts_in, in, opcode, MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
44 err = mlx5_cmd_exec_inout(mdev, query_special_contexts, in, out);
46 *null_mkey = MLX5_GET(query_special_contexts_out, out, null_mkey);
50 static int create_uctx(struct mlx5_vdpa_dev *mvdev, u16 *uid)
52 u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};
57 /* 0 means not supported */
58 if (!MLX5_CAP_GEN(mvdev->mdev, log_max_uctx))
61 inlen = MLX5_ST_SZ_BYTES(create_uctx_in);
62 in = kzalloc(inlen, GFP_KERNEL);
66 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
67 MLX5_SET(create_uctx_in, in, uctx.cap, MLX5_UCTX_CAP_RAW_TX);
69 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out));
72 *uid = MLX5_GET(create_uctx_out, out, uid);
77 static void destroy_uctx(struct mlx5_vdpa_dev *mvdev, u32 uid)
79 u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};
80 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};
82 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
83 MLX5_SET(destroy_uctx_in, in, uid, uid);
85 mlx5_cmd_exec(mvdev->mdev, in, sizeof(in), out, sizeof(out));
88 int mlx5_vdpa_create_tis(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tisn)
90 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {};
93 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
94 MLX5_SET(create_tis_in, in, uid, mvdev->res.uid);
95 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tis, in, out);
97 *tisn = MLX5_GET(create_tis_out, out, tisn);
102 void mlx5_vdpa_destroy_tis(struct mlx5_vdpa_dev *mvdev, u32 tisn)
104 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};
106 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
107 MLX5_SET(destroy_tis_in, in, uid, mvdev->res.uid);
108 MLX5_SET(destroy_tis_in, in, tisn, tisn);
109 mlx5_cmd_exec_in(mvdev->mdev, destroy_tis, in);
112 int mlx5_vdpa_create_rqt(struct mlx5_vdpa_dev *mvdev, void *in, int inlen, u32 *rqtn)
114 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};
117 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
118 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out));
120 *rqtn = MLX5_GET(create_rqt_out, out, rqtn);
125 void mlx5_vdpa_destroy_rqt(struct mlx5_vdpa_dev *mvdev, u32 rqtn)
127 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};
129 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
130 MLX5_SET(destroy_rqt_in, in, uid, mvdev->res.uid);
131 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
132 mlx5_cmd_exec_in(mvdev->mdev, destroy_rqt, in);
135 int mlx5_vdpa_create_tir(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tirn)
137 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
140 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
141 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tir, in, out);
143 *tirn = MLX5_GET(create_tir_out, out, tirn);
148 void mlx5_vdpa_destroy_tir(struct mlx5_vdpa_dev *mvdev, u32 tirn)
150 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};
152 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
153 MLX5_SET(destroy_tir_in, in, uid, mvdev->res.uid);
154 MLX5_SET(destroy_tir_in, in, tirn, tirn);
155 mlx5_cmd_exec_in(mvdev->mdev, destroy_tir, in);
158 int mlx5_vdpa_alloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 *tdn)
160 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};
161 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};
164 MLX5_SET(alloc_transport_domain_in, in, opcode, MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
165 MLX5_SET(alloc_transport_domain_in, in, uid, mvdev->res.uid);
167 err = mlx5_cmd_exec_inout(mvdev->mdev, alloc_transport_domain, in, out);
169 *tdn = MLX5_GET(alloc_transport_domain_out, out, transport_domain);
174 void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 tdn)
176 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};
178 MLX5_SET(dealloc_transport_domain_in, in, opcode, MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
179 MLX5_SET(dealloc_transport_domain_in, in, uid, mvdev->res.uid);
180 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
181 mlx5_cmd_exec_in(mvdev->mdev, dealloc_transport_domain, in);
184 int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey, u32 *in,
187 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {};
192 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
193 MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
195 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, lout, sizeof(lout));
199 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
200 mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index);
201 mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
202 mkey->size = MLX5_GET64(mkc, mkc, len);
203 mkey->key |= mlx5_idx_to_mkey(mkey_index);
204 mkey->pd = MLX5_GET(mkc, mkc, pd);
208 int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey)
210 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {};
212 MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid);
213 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
214 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key));
215 return mlx5_cmd_exec_in(mvdev->mdev, destroy_mkey, in);
218 int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev)
220 u64 offset = MLX5_CAP64_DEV_VDPA_EMULATION(mvdev->mdev, doorbell_bar_offset);
221 struct mlx5_vdpa_resources *res = &mvdev->res;
222 struct mlx5_core_dev *mdev = mvdev->mdev;
227 mlx5_vdpa_warn(mvdev, "resources already allocated\n");
230 mutex_init(&mvdev->mr.mkey_mtx);
231 res->uar = mlx5_get_uars_page(mdev);
232 if (IS_ERR(res->uar)) {
233 err = PTR_ERR(res->uar);
237 err = create_uctx(mvdev, &res->uid);
241 err = alloc_pd(mvdev, &res->pdn, res->uid);
245 err = get_null_mkey(mvdev, &res->null_mkey);
249 kick_addr = pci_resource_start(mdev->pdev, 0) + offset;
250 res->kick_addr = ioremap(kick_addr, PAGE_SIZE);
251 if (!res->kick_addr) {
260 dealloc_pd(mvdev, res->pdn, res->uid);
262 destroy_uctx(mvdev, res->uid);
264 mlx5_put_uars_page(mdev, res->uar);
266 mutex_destroy(&mvdev->mr.mkey_mtx);
270 void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev)
272 struct mlx5_vdpa_resources *res = &mvdev->res;
277 iounmap(res->kick_addr);
278 res->kick_addr = NULL;
279 dealloc_pd(mvdev, res->pdn, res->uid);
280 destroy_uctx(mvdev, res->uid);
281 mlx5_put_uars_page(mvdev->mdev, res->uar);
282 mutex_destroy(&mvdev->mr.mkey_mtx);