GNU Linux-libre 5.19-rc6-gnu
[releases.git] / drivers / cxl / pmem.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
3 #include <linux/libnvdimm.h>
4 #include <asm/unaligned.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/ndctl.h>
8 #include <linux/async.h>
9 #include <linux/slab.h>
10 #include "cxlmem.h"
11 #include "cxl.h"
12
13 /*
14  * Ordered workqueue for cxl nvdimm device arrival and departure
15  * to coordinate bus rescans when a bridge arrives and trigger remove
16  * operations when the bridge is removed.
17  */
18 static struct workqueue_struct *cxl_pmem_wq;
19
20 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
21
22 static void clear_exclusive(void *cxlds)
23 {
24         clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
25 }
26
27 static void unregister_nvdimm(void *nvdimm)
28 {
29         nvdimm_delete(nvdimm);
30 }
31
32 static int cxl_nvdimm_probe(struct device *dev)
33 {
34         struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
35         struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
36         unsigned long flags = 0, cmd_mask = 0;
37         struct cxl_dev_state *cxlds = cxlmd->cxlds;
38         struct cxl_nvdimm_bridge *cxl_nvb;
39         struct nvdimm *nvdimm;
40         int rc;
41
42         cxl_nvb = cxl_find_nvdimm_bridge(cxl_nvd);
43         if (!cxl_nvb)
44                 return -ENXIO;
45
46         device_lock(&cxl_nvb->dev);
47         if (!cxl_nvb->nvdimm_bus) {
48                 rc = -ENXIO;
49                 goto out;
50         }
51
52         set_exclusive_cxl_commands(cxlds, exclusive_cmds);
53         rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
54         if (rc)
55                 goto out;
56
57         set_bit(NDD_LABELING, &flags);
58         set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
59         set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
60         set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
61         nvdimm = nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, NULL, flags,
62                                cmd_mask, 0, NULL);
63         if (!nvdimm) {
64                 rc = -ENOMEM;
65                 goto out;
66         }
67
68         dev_set_drvdata(dev, nvdimm);
69         rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
70 out:
71         device_unlock(&cxl_nvb->dev);
72         put_device(&cxl_nvb->dev);
73
74         return rc;
75 }
76
77 static struct cxl_driver cxl_nvdimm_driver = {
78         .name = "cxl_nvdimm",
79         .probe = cxl_nvdimm_probe,
80         .id = CXL_DEVICE_NVDIMM,
81 };
82
83 static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
84                                     struct nd_cmd_get_config_size *cmd,
85                                     unsigned int buf_len)
86 {
87         if (sizeof(*cmd) > buf_len)
88                 return -EINVAL;
89
90         *cmd = (struct nd_cmd_get_config_size) {
91                  .config_size = cxlds->lsa_size,
92                  .max_xfer = cxlds->payload_size,
93         };
94
95         return 0;
96 }
97
98 static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
99                                     struct nd_cmd_get_config_data_hdr *cmd,
100                                     unsigned int buf_len)
101 {
102         struct cxl_mbox_get_lsa get_lsa;
103         int rc;
104
105         if (sizeof(*cmd) > buf_len)
106                 return -EINVAL;
107         if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
108                 return -EINVAL;
109
110         get_lsa = (struct cxl_mbox_get_lsa) {
111                 .offset = cpu_to_le32(cmd->in_offset),
112                 .length = cpu_to_le32(cmd->in_length),
113         };
114
115         rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_GET_LSA, &get_lsa,
116                                sizeof(get_lsa), cmd->out_buf, cmd->in_length);
117         cmd->status = 0;
118
119         return rc;
120 }
121
122 static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
123                                     struct nd_cmd_set_config_hdr *cmd,
124                                     unsigned int buf_len)
125 {
126         struct cxl_mbox_set_lsa *set_lsa;
127         int rc;
128
129         if (sizeof(*cmd) > buf_len)
130                 return -EINVAL;
131
132         /* 4-byte status follows the input data in the payload */
133         if (struct_size(cmd, in_buf, cmd->in_length) + 4 > buf_len)
134                 return -EINVAL;
135
136         set_lsa =
137                 kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
138         if (!set_lsa)
139                 return -ENOMEM;
140
141         *set_lsa = (struct cxl_mbox_set_lsa) {
142                 .offset = cpu_to_le32(cmd->in_offset),
143         };
144         memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
145
146         rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_SET_LSA, set_lsa,
147                                struct_size(set_lsa, data, cmd->in_length),
148                                NULL, 0);
149
150         /*
151          * Set "firmware" status (4-packed bytes at the end of the input
152          * payload.
153          */
154         put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
155         kvfree(set_lsa);
156
157         return rc;
158 }
159
160 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
161                                void *buf, unsigned int buf_len)
162 {
163         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
164         unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
165         struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
166         struct cxl_dev_state *cxlds = cxlmd->cxlds;
167
168         if (!test_bit(cmd, &cmd_mask))
169                 return -ENOTTY;
170
171         switch (cmd) {
172         case ND_CMD_GET_CONFIG_SIZE:
173                 return cxl_pmem_get_config_size(cxlds, buf, buf_len);
174         case ND_CMD_GET_CONFIG_DATA:
175                 return cxl_pmem_get_config_data(cxlds, buf, buf_len);
176         case ND_CMD_SET_CONFIG_DATA:
177                 return cxl_pmem_set_config_data(cxlds, buf, buf_len);
178         default:
179                 return -ENOTTY;
180         }
181 }
182
183 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
184                         struct nvdimm *nvdimm, unsigned int cmd, void *buf,
185                         unsigned int buf_len, int *cmd_rc)
186 {
187         /*
188          * No firmware response to translate, let the transport error
189          * code take precedence.
190          */
191         *cmd_rc = 0;
192
193         if (!nvdimm)
194                 return -ENOTTY;
195         return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
196 }
197
198 static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb)
199 {
200         if (cxl_nvb->nvdimm_bus)
201                 return true;
202         cxl_nvb->nvdimm_bus =
203                 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
204         return cxl_nvb->nvdimm_bus != NULL;
205 }
206
207 static int cxl_nvdimm_release_driver(struct device *dev, void *data)
208 {
209         if (!is_cxl_nvdimm(dev))
210                 return 0;
211         device_release_driver(dev);
212         return 0;
213 }
214
215 static void offline_nvdimm_bus(struct nvdimm_bus *nvdimm_bus)
216 {
217         if (!nvdimm_bus)
218                 return;
219
220         /*
221          * Set the state of cxl_nvdimm devices to unbound / idle before
222          * nvdimm_bus_unregister() rips the nvdimm objects out from
223          * underneath them.
224          */
225         bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_release_driver);
226         nvdimm_bus_unregister(nvdimm_bus);
227 }
228
229 static void cxl_nvb_update_state(struct work_struct *work)
230 {
231         struct cxl_nvdimm_bridge *cxl_nvb =
232                 container_of(work, typeof(*cxl_nvb), state_work);
233         struct nvdimm_bus *victim_bus = NULL;
234         bool release = false, rescan = false;
235
236         device_lock(&cxl_nvb->dev);
237         switch (cxl_nvb->state) {
238         case CXL_NVB_ONLINE:
239                 if (!online_nvdimm_bus(cxl_nvb)) {
240                         dev_err(&cxl_nvb->dev,
241                                 "failed to establish nvdimm bus\n");
242                         release = true;
243                 } else
244                         rescan = true;
245                 break;
246         case CXL_NVB_OFFLINE:
247         case CXL_NVB_DEAD:
248                 victim_bus = cxl_nvb->nvdimm_bus;
249                 cxl_nvb->nvdimm_bus = NULL;
250                 break;
251         default:
252                 break;
253         }
254         device_unlock(&cxl_nvb->dev);
255
256         if (release)
257                 device_release_driver(&cxl_nvb->dev);
258         if (rescan) {
259                 int rc = bus_rescan_devices(&cxl_bus_type);
260
261                 dev_dbg(&cxl_nvb->dev, "rescan: %d\n", rc);
262         }
263         offline_nvdimm_bus(victim_bus);
264
265         put_device(&cxl_nvb->dev);
266 }
267
268 static void cxl_nvdimm_bridge_state_work(struct cxl_nvdimm_bridge *cxl_nvb)
269 {
270         /*
271          * Take a reference that the workqueue will drop if new work
272          * gets queued.
273          */
274         get_device(&cxl_nvb->dev);
275         if (!queue_work(cxl_pmem_wq, &cxl_nvb->state_work))
276                 put_device(&cxl_nvb->dev);
277 }
278
279 static void cxl_nvdimm_bridge_remove(struct device *dev)
280 {
281         struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
282
283         if (cxl_nvb->state == CXL_NVB_ONLINE)
284                 cxl_nvb->state = CXL_NVB_OFFLINE;
285         cxl_nvdimm_bridge_state_work(cxl_nvb);
286 }
287
288 static int cxl_nvdimm_bridge_probe(struct device *dev)
289 {
290         struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
291
292         if (cxl_nvb->state == CXL_NVB_DEAD)
293                 return -ENXIO;
294
295         if (cxl_nvb->state == CXL_NVB_NEW) {
296                 cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
297                         .provider_name = "CXL",
298                         .module = THIS_MODULE,
299                         .ndctl = cxl_pmem_ctl,
300                 };
301
302                 INIT_WORK(&cxl_nvb->state_work, cxl_nvb_update_state);
303         }
304
305         cxl_nvb->state = CXL_NVB_ONLINE;
306         cxl_nvdimm_bridge_state_work(cxl_nvb);
307
308         return 0;
309 }
310
311 static struct cxl_driver cxl_nvdimm_bridge_driver = {
312         .name = "cxl_nvdimm_bridge",
313         .probe = cxl_nvdimm_bridge_probe,
314         .remove = cxl_nvdimm_bridge_remove,
315         .id = CXL_DEVICE_NVDIMM_BRIDGE,
316 };
317
318 /*
319  * Return all bridges to the CXL_NVB_NEW state to invalidate any
320  * ->state_work referring to the now destroyed cxl_pmem_wq.
321  */
322 static int cxl_nvdimm_bridge_reset(struct device *dev, void *data)
323 {
324         struct cxl_nvdimm_bridge *cxl_nvb;
325
326         if (!is_cxl_nvdimm_bridge(dev))
327                 return 0;
328
329         cxl_nvb = to_cxl_nvdimm_bridge(dev);
330         device_lock(dev);
331         cxl_nvb->state = CXL_NVB_NEW;
332         device_unlock(dev);
333
334         return 0;
335 }
336
337 static void destroy_cxl_pmem_wq(void)
338 {
339         destroy_workqueue(cxl_pmem_wq);
340         bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_bridge_reset);
341 }
342
343 static __init int cxl_pmem_init(void)
344 {
345         int rc;
346
347         set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
348         set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
349
350         cxl_pmem_wq = alloc_ordered_workqueue("cxl_pmem", 0);
351         if (!cxl_pmem_wq)
352                 return -ENXIO;
353
354         rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
355         if (rc)
356                 goto err_bridge;
357
358         rc = cxl_driver_register(&cxl_nvdimm_driver);
359         if (rc)
360                 goto err_nvdimm;
361
362         return 0;
363
364 err_nvdimm:
365         cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
366 err_bridge:
367         destroy_cxl_pmem_wq();
368         return rc;
369 }
370
371 static __exit void cxl_pmem_exit(void)
372 {
373         cxl_driver_unregister(&cxl_nvdimm_driver);
374         cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
375         destroy_cxl_pmem_wq();
376 }
377
378 MODULE_LICENSE("GPL v2");
379 module_init(cxl_pmem_init);
380 module_exit(cxl_pmem_exit);
381 MODULE_IMPORT_NS(CXL);
382 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
383 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);