1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright(c) 2017 Intel Corporation. All rights reserved.
5 #include <linux/pagemap.h>
6 #include <linux/module.h>
7 #include <linux/mount.h>
8 #include <linux/pseudo_fs.h>
9 #include <linux/magic.h>
10 #include <linux/pfn_t.h>
11 #include <linux/cdev.h>
12 #include <linux/slab.h>
13 #include <linux/uio.h>
14 #include <linux/dax.h>
16 #include "dax-private.h"
19 * struct dax_device - anchor object for dax services
21 * @cdev: optional character interface for "device dax"
22 * @private: dax driver private data
23 * @flags: state and boolean properties
24 * @ops: operations for this device
31 const struct dax_operations *ops;
34 static dev_t dax_devt;
35 DEFINE_STATIC_SRCU(dax_srcu);
36 static struct vfsmount *dax_mnt;
37 static DEFINE_IDA(dax_minor_ida);
38 static struct kmem_cache *dax_cache __read_mostly;
39 static struct super_block *dax_superblock __read_mostly;
41 int dax_read_lock(void)
43 return srcu_read_lock(&dax_srcu);
45 EXPORT_SYMBOL_GPL(dax_read_lock);
47 void dax_read_unlock(int id)
49 srcu_read_unlock(&dax_srcu, id);
51 EXPORT_SYMBOL_GPL(dax_read_unlock);
53 #if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
54 #include <linux/blkdev.h>
56 static DEFINE_XARRAY(dax_hosts);
58 int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
60 return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL);
62 EXPORT_SYMBOL_GPL(dax_add_host);
64 void dax_remove_host(struct gendisk *disk)
66 xa_erase(&dax_hosts, (unsigned long)disk);
68 EXPORT_SYMBOL_GPL(dax_remove_host);
71 * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax
72 * @bdev: block device to find a dax_device for
73 * @start_off: returns the byte offset into the dax_device that @bdev starts
75 struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off)
77 struct dax_device *dax_dev;
81 if (!blk_queue_dax(bdev->bd_disk->queue))
84 *start_off = get_start_sect(bdev) * SECTOR_SIZE;
85 part_size = bdev_nr_sectors(bdev) * SECTOR_SIZE;
86 if (*start_off % PAGE_SIZE || part_size % PAGE_SIZE) {
87 pr_info("%pg: error: unaligned partition for dax\n", bdev);
92 dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk);
93 if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode))
99 EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);
100 #endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
102 enum dax_device_flags {
103 /* !alive + rcu grace period == no new operations / mappings */
105 /* gate whether dax_flush() calls the low level flush routine */
107 /* flag to check if device supports synchronous flush */
109 /* do not leave the caches dirty after writes */
111 /* handle CPU fetch exceptions during reads */
116 * dax_direct_access() - translate a device pgoff to an absolute pfn
117 * @dax_dev: a dax_device instance representing the logical memory range
118 * @pgoff: offset in pages from the start of the device to translate
119 * @nr_pages: number of consecutive pages caller can handle relative to @pfn
120 * @mode: indicator on normal access or recovery write
121 * @kaddr: output parameter that returns a virtual address mapping of pfn
122 * @pfn: output parameter that returns an absolute pfn translation of @pgoff
124 * Return: negative errno if an error occurs, otherwise the number of
125 * pages accessible at the device relative @pgoff.
127 long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
128 enum dax_access_mode mode, void **kaddr, pfn_t *pfn)
135 if (!dax_alive(dax_dev))
141 avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,
145 return min(avail, nr_pages);
147 EXPORT_SYMBOL_GPL(dax_direct_access);
149 size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
150 size_t bytes, struct iov_iter *i)
152 if (!dax_alive(dax_dev))
156 * The userspace address for the memory copy has already been validated
157 * via access_ok() in vfs_write, so use the 'no check' version to bypass
158 * the HARDENED_USERCOPY overhead.
160 if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags))
161 return _copy_from_iter_flushcache(addr, bytes, i);
162 return _copy_from_iter(addr, bytes, i);
165 size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
166 size_t bytes, struct iov_iter *i)
168 if (!dax_alive(dax_dev))
172 * The userspace address for the memory copy has already been validated
173 * via access_ok() in vfs_red, so use the 'no check' version to bypass
174 * the HARDENED_USERCOPY overhead.
176 if (test_bit(DAXDEV_NOMC, &dax_dev->flags))
177 return _copy_mc_to_iter(addr, bytes, i);
178 return _copy_to_iter(addr, bytes, i);
181 int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
184 if (!dax_alive(dax_dev))
187 * There are no callers that want to zero more than one page as of now.
188 * Once users are there, this check can be removed after the
189 * device mapper code has been updated to split ranges across targets.
194 return dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);
196 EXPORT_SYMBOL_GPL(dax_zero_page_range);
198 size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
199 void *addr, size_t bytes, struct iov_iter *iter)
201 if (!dax_dev->ops->recovery_write)
203 return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);
205 EXPORT_SYMBOL_GPL(dax_recovery_write);
207 #ifdef CONFIG_ARCH_HAS_PMEM_API
208 void arch_wb_cache_pmem(void *addr, size_t size);
209 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
211 if (unlikely(!dax_write_cache_enabled(dax_dev)))
214 arch_wb_cache_pmem(addr, size);
217 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
221 EXPORT_SYMBOL_GPL(dax_flush);
223 void dax_write_cache(struct dax_device *dax_dev, bool wc)
226 set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
228 clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
230 EXPORT_SYMBOL_GPL(dax_write_cache);
232 bool dax_write_cache_enabled(struct dax_device *dax_dev)
234 return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
236 EXPORT_SYMBOL_GPL(dax_write_cache_enabled);
238 bool dax_synchronous(struct dax_device *dax_dev)
240 return test_bit(DAXDEV_SYNC, &dax_dev->flags);
242 EXPORT_SYMBOL_GPL(dax_synchronous);
244 void set_dax_synchronous(struct dax_device *dax_dev)
246 set_bit(DAXDEV_SYNC, &dax_dev->flags);
248 EXPORT_SYMBOL_GPL(set_dax_synchronous);
250 void set_dax_nocache(struct dax_device *dax_dev)
252 set_bit(DAXDEV_NOCACHE, &dax_dev->flags);
254 EXPORT_SYMBOL_GPL(set_dax_nocache);
256 void set_dax_nomc(struct dax_device *dax_dev)
258 set_bit(DAXDEV_NOMC, &dax_dev->flags);
260 EXPORT_SYMBOL_GPL(set_dax_nomc);
262 bool dax_alive(struct dax_device *dax_dev)
264 lockdep_assert_held(&dax_srcu);
265 return test_bit(DAXDEV_ALIVE, &dax_dev->flags);
267 EXPORT_SYMBOL_GPL(dax_alive);
270 * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring
271 * that any fault handlers or operations that might have seen
272 * dax_alive(), have completed. Any operations that start after
273 * synchronize_srcu() has run will abort upon seeing !dax_alive().
275 void kill_dax(struct dax_device *dax_dev)
280 clear_bit(DAXDEV_ALIVE, &dax_dev->flags);
281 synchronize_srcu(&dax_srcu);
283 EXPORT_SYMBOL_GPL(kill_dax);
285 void run_dax(struct dax_device *dax_dev)
287 set_bit(DAXDEV_ALIVE, &dax_dev->flags);
289 EXPORT_SYMBOL_GPL(run_dax);
291 static struct inode *dax_alloc_inode(struct super_block *sb)
293 struct dax_device *dax_dev;
296 dax_dev = alloc_inode_sb(sb, dax_cache, GFP_KERNEL);
300 inode = &dax_dev->inode;
305 static struct dax_device *to_dax_dev(struct inode *inode)
307 return container_of(inode, struct dax_device, inode);
310 static void dax_free_inode(struct inode *inode)
312 struct dax_device *dax_dev = to_dax_dev(inode);
314 ida_simple_remove(&dax_minor_ida, iminor(inode));
315 kmem_cache_free(dax_cache, dax_dev);
318 static void dax_destroy_inode(struct inode *inode)
320 struct dax_device *dax_dev = to_dax_dev(inode);
321 WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),
322 "kill_dax() must be called before final iput()\n");
325 static const struct super_operations dax_sops = {
326 .statfs = simple_statfs,
327 .alloc_inode = dax_alloc_inode,
328 .destroy_inode = dax_destroy_inode,
329 .free_inode = dax_free_inode,
330 .drop_inode = generic_delete_inode,
333 static int dax_init_fs_context(struct fs_context *fc)
335 struct pseudo_fs_context *ctx = init_pseudo(fc, DAXFS_MAGIC);
338 ctx->ops = &dax_sops;
342 static struct file_system_type dax_fs_type = {
344 .init_fs_context = dax_init_fs_context,
345 .kill_sb = kill_anon_super,
348 static int dax_test(struct inode *inode, void *data)
350 dev_t devt = *(dev_t *) data;
352 return inode->i_rdev == devt;
355 static int dax_set(struct inode *inode, void *data)
357 dev_t devt = *(dev_t *) data;
359 inode->i_rdev = devt;
363 static struct dax_device *dax_dev_get(dev_t devt)
365 struct dax_device *dax_dev;
368 inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
369 dax_test, dax_set, &devt);
374 dax_dev = to_dax_dev(inode);
375 if (inode->i_state & I_NEW) {
376 set_bit(DAXDEV_ALIVE, &dax_dev->flags);
377 inode->i_cdev = &dax_dev->cdev;
378 inode->i_mode = S_IFCHR;
379 inode->i_flags = S_DAX;
380 mapping_set_gfp_mask(&inode->i_data, GFP_USER);
381 unlock_new_inode(inode);
387 struct dax_device *alloc_dax(void *private, const struct dax_operations *ops)
389 struct dax_device *dax_dev;
393 if (WARN_ON_ONCE(ops && !ops->zero_page_range))
394 return ERR_PTR(-EINVAL);
396 minor = ida_simple_get(&dax_minor_ida, 0, MINORMASK+1, GFP_KERNEL);
398 return ERR_PTR(-ENOMEM);
400 devt = MKDEV(MAJOR(dax_devt), minor);
401 dax_dev = dax_dev_get(devt);
406 dax_dev->private = private;
410 ida_simple_remove(&dax_minor_ida, minor);
411 return ERR_PTR(-ENOMEM);
413 EXPORT_SYMBOL_GPL(alloc_dax);
415 void put_dax(struct dax_device *dax_dev)
419 iput(&dax_dev->inode);
421 EXPORT_SYMBOL_GPL(put_dax);
424 * inode_dax: convert a public inode into its dax_dev
425 * @inode: An inode with i_cdev pointing to a dax_dev
427 * Note this is not equivalent to to_dax_dev() which is for private
428 * internal use where we know the inode filesystem type == dax_fs_type.
430 struct dax_device *inode_dax(struct inode *inode)
432 struct cdev *cdev = inode->i_cdev;
434 return container_of(cdev, struct dax_device, cdev);
436 EXPORT_SYMBOL_GPL(inode_dax);
438 struct inode *dax_inode(struct dax_device *dax_dev)
440 return &dax_dev->inode;
442 EXPORT_SYMBOL_GPL(dax_inode);
444 void *dax_get_private(struct dax_device *dax_dev)
446 if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))
448 return dax_dev->private;
450 EXPORT_SYMBOL_GPL(dax_get_private);
452 static void init_once(void *_dax_dev)
454 struct dax_device *dax_dev = _dax_dev;
455 struct inode *inode = &dax_dev->inode;
457 memset(dax_dev, 0, sizeof(*dax_dev));
458 inode_init_once(inode);
461 static int dax_fs_init(void)
465 dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0,
466 (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
467 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
472 dax_mnt = kern_mount(&dax_fs_type);
473 if (IS_ERR(dax_mnt)) {
474 rc = PTR_ERR(dax_mnt);
477 dax_superblock = dax_mnt->mnt_sb;
482 kmem_cache_destroy(dax_cache);
487 static void dax_fs_exit(void)
489 kern_unmount(dax_mnt);
491 kmem_cache_destroy(dax_cache);
494 static int __init dax_core_init(void)
502 rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax");
512 unregister_chrdev_region(dax_devt, MINORMASK+1);
518 static void __exit dax_core_exit(void)
521 unregister_chrdev_region(dax_devt, MINORMASK+1);
522 ida_destroy(&dax_minor_ida);
526 MODULE_AUTHOR("Intel Corporation");
527 MODULE_LICENSE("GPL v2");
528 subsys_initcall(dax_core_init);
529 module_exit(dax_core_exit);