1 // SPDX-License-Identifier: GPL-2.0-only
3 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
5 * bitmap_create - sets up the bitmap structure
6 * bitmap_destroy - destroys the bitmap structure
8 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
9 * - added disk storage for bitmap
10 * - changes to allow various bitmap chunk sizes
16 * flush after percent set rather than just time based. (maybe both).
19 #include <linux/blkdev.h>
20 #include <linux/module.h>
21 #include <linux/errno.h>
22 #include <linux/slab.h>
23 #include <linux/init.h>
24 #include <linux/timer.h>
25 #include <linux/sched.h>
26 #include <linux/list.h>
27 #include <linux/file.h>
28 #include <linux/mount.h>
29 #include <linux/buffer_head.h>
30 #include <linux/seq_file.h>
31 #include <trace/events/block.h>
33 #include "md-bitmap.h"
35 static inline char *bmname(struct bitmap *bitmap)
37 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
41 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
43 * 1) check to see if this page is allocated, if it's not then try to alloc
44 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
45 * page pointer directly as a counter
47 * if we find our page, we increment the page's refcount so that it stays
48 * allocated while we're using it
50 static int md_bitmap_checkpage(struct bitmap_counts *bitmap,
51 unsigned long page, int create, int no_hijack)
52 __releases(bitmap->lock)
53 __acquires(bitmap->lock)
55 unsigned char *mappage;
57 WARN_ON_ONCE(page >= bitmap->pages);
58 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
61 if (bitmap->bp[page].map) /* page is already allocated, just return */
67 /* this page has not been allocated yet */
69 spin_unlock_irq(&bitmap->lock);
70 /* It is possible that this is being called inside a
71 * prepare_to_wait/finish_wait loop from raid5c:make_request().
72 * In general it is not permitted to sleep in that context as it
73 * can cause the loop to spin freely.
74 * That doesn't apply here as we can only reach this point
76 * When this function completes, either bp[page].map or
77 * bp[page].hijacked. In either case, this function will
78 * abort before getting to this point again. So there is
79 * no risk of a free-spin, and so it is safe to assert
80 * that sleeping here is allowed.
82 sched_annotate_sleep();
83 mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
84 spin_lock_irq(&bitmap->lock);
86 if (mappage == NULL) {
87 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
88 /* We don't support hijack for cluster raid */
91 /* failed - set the hijacked flag so that we can use the
92 * pointer as a counter */
93 if (!bitmap->bp[page].map)
94 bitmap->bp[page].hijacked = 1;
95 } else if (bitmap->bp[page].map ||
96 bitmap->bp[page].hijacked) {
97 /* somebody beat us to getting the page */
101 /* no page was in place and we have one, so install it */
103 bitmap->bp[page].map = mappage;
104 bitmap->missing_pages--;
109 /* if page is completely empty, put it back on the free list, or dealloc it */
110 /* if page was hijacked, unmark the flag so it might get alloced next time */
111 /* Note: lock should be held when calling this */
112 static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
116 if (bitmap->bp[page].count) /* page is still busy */
119 /* page is no longer in use, it can be released */
121 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
122 bitmap->bp[page].hijacked = 0;
123 bitmap->bp[page].map = NULL;
125 /* normal case, free the page */
126 ptr = bitmap->bp[page].map;
127 bitmap->bp[page].map = NULL;
128 bitmap->missing_pages++;
134 * bitmap file handling - read and write the bitmap file and its superblock
138 * basic page I/O operations
141 /* IO operations when bitmap is stored near all superblocks */
142 static int read_sb_page(struct mddev *mddev, loff_t offset,
144 unsigned long index, int size)
146 /* choose a good rdev and read the page from there */
148 struct md_rdev *rdev;
151 rdev_for_each(rdev, mddev) {
152 if (! test_bit(In_sync, &rdev->flags)
153 || test_bit(Faulty, &rdev->flags)
154 || test_bit(Bitmap_sync, &rdev->flags))
157 target = offset + index * (PAGE_SIZE/512);
159 if (sync_page_io(rdev, target,
160 roundup(size, bdev_logical_block_size(rdev->bdev)),
161 page, REQ_OP_READ, 0, true)) {
169 static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
171 /* Iterate the disks of an mddev, using rcu to protect access to the
172 * linked list, and raising the refcount of devices we return to ensure
173 * they don't disappear while in use.
174 * As devices are only added or removed when raid_disk is < 0 and
175 * nr_pending is 0 and In_sync is clear, the entries we return will
176 * still be in the same position on the list when we re-enter
177 * list_for_each_entry_continue_rcu.
179 * Note that if entered with 'rdev == NULL' to start at the
180 * beginning, we temporarily assign 'rdev' to an address which
181 * isn't really an rdev, but which can be used by
182 * list_for_each_entry_continue_rcu() to find the first entry.
186 /* start at the beginning */
187 rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
189 /* release the previous rdev and start from there. */
190 rdev_dec_pending(rdev, mddev);
192 list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
193 if (rdev->raid_disk >= 0 &&
194 !test_bit(Faulty, &rdev->flags)) {
195 /* this is a usable devices */
196 atomic_inc(&rdev->nr_pending);
205 static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
207 struct md_rdev *rdev;
208 struct block_device *bdev;
209 struct mddev *mddev = bitmap->mddev;
210 struct bitmap_storage *store = &bitmap->storage;
214 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
215 int size = PAGE_SIZE;
216 loff_t offset = mddev->bitmap_info.offset;
218 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
220 if (page->index == store->file_pages-1) {
221 int last_page_size = store->bytes & (PAGE_SIZE-1);
222 if (last_page_size == 0)
223 last_page_size = PAGE_SIZE;
224 size = roundup(last_page_size,
225 bdev_logical_block_size(bdev));
227 /* Just make sure we aren't corrupting data or
230 if (mddev->external) {
231 /* Bitmap could be anywhere. */
232 if (rdev->sb_start + offset + (page->index
236 rdev->sb_start + offset
237 < (rdev->data_offset + mddev->dev_sectors
240 } else if (offset < 0) {
241 /* DATA BITMAP METADATA */
243 + (long)(page->index * (PAGE_SIZE/512))
245 /* bitmap runs in to metadata */
247 if (rdev->data_offset + mddev->dev_sectors
248 > rdev->sb_start + offset)
249 /* data runs in to bitmap */
251 } else if (rdev->sb_start < rdev->data_offset) {
252 /* METADATA BITMAP DATA */
255 + page->index*(PAGE_SIZE/512) + size/512
257 /* bitmap runs in to data */
260 /* DATA METADATA BITMAP - no problems */
262 md_super_write(mddev, rdev,
263 rdev->sb_start + offset
264 + page->index * (PAGE_SIZE/512),
269 if (wait && md_super_wait(mddev) < 0)
277 static void md_bitmap_file_kick(struct bitmap *bitmap);
279 * write out a page to a file
281 static void write_page(struct bitmap *bitmap, struct page *page, int wait)
283 struct buffer_head *bh;
285 if (bitmap->storage.file == NULL) {
286 switch (write_sb_page(bitmap, page, wait)) {
288 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
292 bh = page_buffers(page);
294 while (bh && bh->b_blocknr) {
295 atomic_inc(&bitmap->pending_writes);
296 set_buffer_locked(bh);
297 set_buffer_mapped(bh);
298 submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
299 bh = bh->b_this_page;
303 wait_event(bitmap->write_wait,
304 atomic_read(&bitmap->pending_writes)==0);
306 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
307 md_bitmap_file_kick(bitmap);
310 static void end_bitmap_write(struct buffer_head *bh, int uptodate)
312 struct bitmap *bitmap = bh->b_private;
315 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
316 if (atomic_dec_and_test(&bitmap->pending_writes))
317 wake_up(&bitmap->write_wait);
320 static void free_buffers(struct page *page)
322 struct buffer_head *bh;
324 if (!PagePrivate(page))
327 bh = page_buffers(page);
329 struct buffer_head *next = bh->b_this_page;
330 free_buffer_head(bh);
333 detach_page_private(page);
337 /* read a page from a file.
338 * We both read the page, and attach buffers to the page to record the
339 * address of each block (using bmap). These addresses will be used
340 * to write the block later, completely bypassing the filesystem.
341 * This usage is similar to how swap files are handled, and allows us
342 * to write to a file with no concerns of memory allocation failing.
344 static int read_page(struct file *file, unsigned long index,
345 struct bitmap *bitmap,
350 struct inode *inode = file_inode(file);
351 struct buffer_head *bh;
352 sector_t block, blk_cur;
353 unsigned long blocksize = i_blocksize(inode);
355 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
356 (unsigned long long)index << PAGE_SHIFT);
358 bh = alloc_page_buffers(page, blocksize, false);
363 attach_page_private(page, bh);
364 blk_cur = index << (PAGE_SHIFT - inode->i_blkbits);
371 ret = bmap(inode, &block);
378 bh->b_blocknr = block;
379 bh->b_bdev = inode->i_sb->s_bdev;
380 if (count < blocksize)
385 bh->b_end_io = end_bitmap_write;
386 bh->b_private = bitmap;
387 atomic_inc(&bitmap->pending_writes);
388 set_buffer_locked(bh);
389 set_buffer_mapped(bh);
390 submit_bh(REQ_OP_READ, 0, bh);
393 bh = bh->b_this_page;
397 wait_event(bitmap->write_wait,
398 atomic_read(&bitmap->pending_writes)==0);
399 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
403 pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
405 (unsigned long long)index << PAGE_SHIFT,
411 * bitmap file superblock operations
415 * md_bitmap_wait_writes() should be called before writing any bitmap
416 * blocks, to ensure previous writes, particularly from
417 * md_bitmap_daemon_work(), have completed.
419 static void md_bitmap_wait_writes(struct bitmap *bitmap)
421 if (bitmap->storage.file)
422 wait_event(bitmap->write_wait,
423 atomic_read(&bitmap->pending_writes)==0);
425 /* Note that we ignore the return value. The writes
426 * might have failed, but that would just mean that
427 * some bits which should be cleared haven't been,
428 * which is safe. The relevant bitmap blocks will
429 * probably get written again, but there is no great
430 * loss if they aren't.
432 md_super_wait(bitmap->mddev);
436 /* update the event counter and sync the superblock to disk */
437 void md_bitmap_update_sb(struct bitmap *bitmap)
441 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
443 if (bitmap->mddev->bitmap_info.external)
445 if (!bitmap->storage.sb_page) /* no superblock */
447 sb = kmap_atomic(bitmap->storage.sb_page);
448 sb->events = cpu_to_le64(bitmap->mddev->events);
449 if (bitmap->mddev->events < bitmap->events_cleared)
450 /* rocking back to read-only */
451 bitmap->events_cleared = bitmap->mddev->events;
452 sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
454 * clear BITMAP_WRITE_ERROR bit to protect against the case that
455 * a bitmap write error occurred but the later writes succeeded.
457 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR));
458 /* Just in case these have been changed via sysfs: */
459 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
460 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
461 /* This might have been changed by a reshape */
462 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
463 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
464 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
465 sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
468 write_page(bitmap, bitmap->storage.sb_page, 1);
470 EXPORT_SYMBOL(md_bitmap_update_sb);
472 /* print out the bitmap file superblock */
473 void md_bitmap_print_sb(struct bitmap *bitmap)
477 if (!bitmap || !bitmap->storage.sb_page)
479 sb = kmap_atomic(bitmap->storage.sb_page);
480 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
481 pr_debug(" magic: %08x\n", le32_to_cpu(sb->magic));
482 pr_debug(" version: %u\n", le32_to_cpu(sb->version));
483 pr_debug(" uuid: %08x.%08x.%08x.%08x\n",
484 le32_to_cpu(*(__le32 *)(sb->uuid+0)),
485 le32_to_cpu(*(__le32 *)(sb->uuid+4)),
486 le32_to_cpu(*(__le32 *)(sb->uuid+8)),
487 le32_to_cpu(*(__le32 *)(sb->uuid+12)));
488 pr_debug(" events: %llu\n",
489 (unsigned long long) le64_to_cpu(sb->events));
490 pr_debug("events cleared: %llu\n",
491 (unsigned long long) le64_to_cpu(sb->events_cleared));
492 pr_debug(" state: %08x\n", le32_to_cpu(sb->state));
493 pr_debug(" chunksize: %u B\n", le32_to_cpu(sb->chunksize));
494 pr_debug(" daemon sleep: %us\n", le32_to_cpu(sb->daemon_sleep));
495 pr_debug(" sync size: %llu KB\n",
496 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
497 pr_debug("max write behind: %u\n", le32_to_cpu(sb->write_behind));
505 * This function is somewhat the reverse of bitmap_read_sb. bitmap_read_sb
506 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
507 * This function verifies 'bitmap_info' and populates the on-disk bitmap
508 * structure, which is to be written to disk.
510 * Returns: 0 on success, -Exxx on error
512 static int md_bitmap_new_disk_sb(struct bitmap *bitmap)
515 unsigned long chunksize, daemon_sleep, write_behind;
517 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
518 if (bitmap->storage.sb_page == NULL)
520 bitmap->storage.sb_page->index = 0;
522 sb = kmap_atomic(bitmap->storage.sb_page);
524 sb->magic = cpu_to_le32(BITMAP_MAGIC);
525 sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
527 chunksize = bitmap->mddev->bitmap_info.chunksize;
529 if (!is_power_of_2(chunksize)) {
531 pr_warn("bitmap chunksize not a power of 2\n");
534 sb->chunksize = cpu_to_le32(chunksize);
536 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
537 if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
538 pr_debug("Choosing daemon_sleep default (5 sec)\n");
539 daemon_sleep = 5 * HZ;
541 sb->daemon_sleep = cpu_to_le32(daemon_sleep);
542 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
545 * FIXME: write_behind for RAID1. If not specified, what
546 * is a good choice? We choose COUNTER_MAX / 2 arbitrarily.
548 write_behind = bitmap->mddev->bitmap_info.max_write_behind;
549 if (write_behind > COUNTER_MAX)
550 write_behind = COUNTER_MAX / 2;
551 sb->write_behind = cpu_to_le32(write_behind);
552 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
554 /* keep the array size field of the bitmap superblock up to date */
555 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
557 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
559 set_bit(BITMAP_STALE, &bitmap->flags);
560 sb->state = cpu_to_le32(bitmap->flags);
561 bitmap->events_cleared = bitmap->mddev->events;
562 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
563 bitmap->mddev->bitmap_info.nodes = 0;
570 /* read the superblock from the bitmap file and initialize some bitmap fields */
571 static int md_bitmap_read_sb(struct bitmap *bitmap)
575 unsigned long chunksize, daemon_sleep, write_behind;
576 unsigned long long events;
578 unsigned long sectors_reserved = 0;
580 struct page *sb_page;
581 loff_t offset = bitmap->mddev->bitmap_info.offset;
583 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
584 chunksize = 128 * 1024 * 1024;
585 daemon_sleep = 5 * HZ;
587 set_bit(BITMAP_STALE, &bitmap->flags);
591 /* page 0 is the superblock, read it... */
592 sb_page = alloc_page(GFP_KERNEL);
595 bitmap->storage.sb_page = sb_page;
598 /* If cluster_slot is set, the cluster is setup */
599 if (bitmap->cluster_slot >= 0) {
600 sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
602 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks,
603 (bitmap->mddev->bitmap_info.chunksize >> 9));
605 bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
607 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
608 offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3));
609 pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
610 bitmap->cluster_slot, offset);
613 if (bitmap->storage.file) {
614 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
615 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
617 err = read_page(bitmap->storage.file, 0,
618 bitmap, bytes, sb_page);
620 err = read_sb_page(bitmap->mddev,
623 0, sizeof(bitmap_super_t));
629 sb = kmap_atomic(sb_page);
631 chunksize = le32_to_cpu(sb->chunksize);
632 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
633 write_behind = le32_to_cpu(sb->write_behind);
634 sectors_reserved = le32_to_cpu(sb->sectors_reserved);
636 /* verify that the bitmap-specific fields are valid */
637 if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
638 reason = "bad magic";
639 else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
640 le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
641 reason = "unrecognized superblock version";
642 else if (chunksize < 512)
643 reason = "bitmap chunksize too small";
644 else if (!is_power_of_2(chunksize))
645 reason = "bitmap chunksize not a power of 2";
646 else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
647 reason = "daemon sleep period out of range";
648 else if (write_behind > COUNTER_MAX)
649 reason = "write-behind limit out of range (0 - 16383)";
651 pr_warn("%s: invalid bitmap file superblock: %s\n",
652 bmname(bitmap), reason);
657 * Setup nodes/clustername only if bitmap version is
660 if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
661 nodes = le32_to_cpu(sb->nodes);
662 strlcpy(bitmap->mddev->bitmap_info.cluster_name,
663 sb->cluster_name, 64);
666 /* keep the array size field of the bitmap superblock up to date */
667 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
669 if (bitmap->mddev->persistent) {
671 * We have a persistent array superblock, so compare the
672 * bitmap's UUID and event counter to the mddev's
674 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
675 pr_warn("%s: bitmap superblock UUID mismatch\n",
679 events = le64_to_cpu(sb->events);
680 if (!nodes && (events < bitmap->mddev->events)) {
681 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
682 bmname(bitmap), events,
683 (unsigned long long) bitmap->mddev->events);
684 set_bit(BITMAP_STALE, &bitmap->flags);
688 /* assign fields using values from superblock */
689 bitmap->flags |= le32_to_cpu(sb->state);
690 if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
691 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
692 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
693 strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
698 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
699 /* Assigning chunksize is required for "re_read" */
700 bitmap->mddev->bitmap_info.chunksize = chunksize;
701 err = md_setup_cluster(bitmap->mddev, nodes);
703 pr_warn("%s: Could not setup cluster service (%d)\n",
704 bmname(bitmap), err);
707 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
713 if (test_bit(BITMAP_STALE, &bitmap->flags))
714 bitmap->events_cleared = bitmap->mddev->events;
715 bitmap->mddev->bitmap_info.chunksize = chunksize;
716 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
717 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
718 bitmap->mddev->bitmap_info.nodes = nodes;
719 if (bitmap->mddev->bitmap_info.space == 0 ||
720 bitmap->mddev->bitmap_info.space > sectors_reserved)
721 bitmap->mddev->bitmap_info.space = sectors_reserved;
723 md_bitmap_print_sb(bitmap);
724 if (bitmap->cluster_slot < 0)
725 md_cluster_stop(bitmap->mddev);
731 * general bitmap file operations
737 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
738 * file a page at a time. There's a superblock at the start of the file.
740 /* calculate the index of the page that contains this bit */
741 static inline unsigned long file_page_index(struct bitmap_storage *store,
745 chunk += sizeof(bitmap_super_t) << 3;
746 return chunk >> PAGE_BIT_SHIFT;
749 /* calculate the (bit) offset of this bit within a page */
750 static inline unsigned long file_page_offset(struct bitmap_storage *store,
754 chunk += sizeof(bitmap_super_t) << 3;
755 return chunk & (PAGE_BITS - 1);
759 * return a pointer to the page in the filemap that contains the given bit
762 static inline struct page *filemap_get_page(struct bitmap_storage *store,
765 if (file_page_index(store, chunk) >= store->file_pages)
767 return store->filemap[file_page_index(store, chunk)];
770 static int md_bitmap_storage_alloc(struct bitmap_storage *store,
771 unsigned long chunks, int with_super,
774 int pnum, offset = 0;
775 unsigned long num_pages;
778 bytes = DIV_ROUND_UP(chunks, 8);
780 bytes += sizeof(bitmap_super_t);
782 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
783 offset = slot_number * num_pages;
785 store->filemap = kmalloc_array(num_pages, sizeof(struct page *),
790 if (with_super && !store->sb_page) {
791 store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
792 if (store->sb_page == NULL)
797 if (store->sb_page) {
798 store->filemap[0] = store->sb_page;
800 store->sb_page->index = offset;
803 for ( ; pnum < num_pages; pnum++) {
804 store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
805 if (!store->filemap[pnum]) {
806 store->file_pages = pnum;
809 store->filemap[pnum]->index = pnum + offset;
811 store->file_pages = pnum;
813 /* We need 4 bits per page, rounded up to a multiple
814 * of sizeof(unsigned long) */
815 store->filemap_attr = kzalloc(
816 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
818 if (!store->filemap_attr)
821 store->bytes = bytes;
826 static void md_bitmap_file_unmap(struct bitmap_storage *store)
828 struct page **map, *sb_page;
833 map = store->filemap;
834 pages = store->file_pages;
835 sb_page = store->sb_page;
838 if (map[pages] != sb_page) /* 0 is sb_page, release it below */
839 free_buffers(map[pages]);
841 kfree(store->filemap_attr);
844 free_buffers(sb_page);
847 struct inode *inode = file_inode(file);
848 invalidate_mapping_pages(inode->i_mapping, 0, -1);
854 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
855 * then it is no longer reliable, so we stop using it and we mark the file
856 * as failed in the superblock
858 static void md_bitmap_file_kick(struct bitmap *bitmap)
860 char *path, *ptr = NULL;
862 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
863 md_bitmap_update_sb(bitmap);
865 if (bitmap->storage.file) {
866 path = kmalloc(PAGE_SIZE, GFP_KERNEL);
868 ptr = file_path(bitmap->storage.file,
871 pr_warn("%s: kicking failed bitmap file %s from array!\n",
872 bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
876 pr_warn("%s: disabling internal bitmap due to errors\n",
881 enum bitmap_page_attr {
882 BITMAP_PAGE_DIRTY = 0, /* there are set bits that need to be synced */
883 BITMAP_PAGE_PENDING = 1, /* there are bits that are being cleaned.
884 * i.e. counter is 1 or 2. */
885 BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
888 static inline void set_page_attr(struct bitmap *bitmap, int pnum,
889 enum bitmap_page_attr attr)
891 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
894 static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
895 enum bitmap_page_attr attr)
897 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
900 static inline int test_page_attr(struct bitmap *bitmap, int pnum,
901 enum bitmap_page_attr attr)
903 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
906 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
907 enum bitmap_page_attr attr)
909 return test_and_clear_bit((pnum<<2) + attr,
910 bitmap->storage.filemap_attr);
913 * bitmap_file_set_bit -- called before performing a write to the md device
914 * to set (and eventually sync) a particular bit in the bitmap file
916 * we set the bit immediately, then we record the page number so that
917 * when an unplug occurs, we can flush the dirty pages out to disk
919 static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
924 unsigned long chunk = block >> bitmap->counts.chunkshift;
925 struct bitmap_storage *store = &bitmap->storage;
926 unsigned long node_offset = 0;
928 if (mddev_is_clustered(bitmap->mddev))
929 node_offset = bitmap->cluster_slot * store->file_pages;
931 page = filemap_get_page(&bitmap->storage, chunk);
934 bit = file_page_offset(&bitmap->storage, chunk);
937 kaddr = kmap_atomic(page);
938 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
941 set_bit_le(bit, kaddr);
942 kunmap_atomic(kaddr);
943 pr_debug("set file bit %lu page %lu\n", bit, page->index);
944 /* record page number so it gets flushed to disk when unplug occurs */
945 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_DIRTY);
948 static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
953 unsigned long chunk = block >> bitmap->counts.chunkshift;
954 struct bitmap_storage *store = &bitmap->storage;
955 unsigned long node_offset = 0;
957 if (mddev_is_clustered(bitmap->mddev))
958 node_offset = bitmap->cluster_slot * store->file_pages;
960 page = filemap_get_page(&bitmap->storage, chunk);
963 bit = file_page_offset(&bitmap->storage, chunk);
964 paddr = kmap_atomic(page);
965 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
966 clear_bit(bit, paddr);
968 clear_bit_le(bit, paddr);
969 kunmap_atomic(paddr);
970 if (!test_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
971 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_PENDING);
972 bitmap->allclean = 0;
976 static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
981 unsigned long chunk = block >> bitmap->counts.chunkshift;
984 page = filemap_get_page(&bitmap->storage, chunk);
987 bit = file_page_offset(&bitmap->storage, chunk);
988 paddr = kmap_atomic(page);
989 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
990 set = test_bit(bit, paddr);
992 set = test_bit_le(bit, paddr);
993 kunmap_atomic(paddr);
998 /* this gets called when the md device is ready to unplug its underlying
999 * (slave) device queues -- before we let any writes go down, we need to
1000 * sync the dirty pages of the bitmap file to disk */
1001 void md_bitmap_unplug(struct bitmap *bitmap)
1004 int dirty, need_write;
1007 if (!bitmap || !bitmap->storage.filemap ||
1008 test_bit(BITMAP_STALE, &bitmap->flags))
1011 /* look at each page to see if there are any set bits that need to be
1012 * flushed out to disk */
1013 for (i = 0; i < bitmap->storage.file_pages; i++) {
1014 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1015 need_write = test_and_clear_page_attr(bitmap, i,
1016 BITMAP_PAGE_NEEDWRITE);
1017 if (dirty || need_write) {
1019 md_bitmap_wait_writes(bitmap);
1020 if (bitmap->mddev->queue)
1021 blk_add_trace_msg(bitmap->mddev->queue,
1022 "md bitmap_unplug");
1024 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
1025 write_page(bitmap, bitmap->storage.filemap[i], 0);
1030 md_bitmap_wait_writes(bitmap);
1032 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1033 md_bitmap_file_kick(bitmap);
1035 EXPORT_SYMBOL(md_bitmap_unplug);
1037 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1038 /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
1039 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
1040 * memory mapping of the bitmap file
1042 * if there's no bitmap file, or if the bitmap file had been
1043 * previously kicked from the array, we mark all the bits as
1044 * 1's in order to cause a full resync.
1046 * We ignore all bits for sectors that end earlier than 'start'.
1047 * This is used when reading an out-of-date bitmap...
1049 static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1051 unsigned long i, chunks, index, oldindex, bit, node_offset = 0;
1052 struct page *page = NULL;
1053 unsigned long bit_cnt = 0;
1055 unsigned long offset;
1059 struct bitmap_storage *store = &bitmap->storage;
1061 chunks = bitmap->counts.chunks;
1064 if (!file && !bitmap->mddev->bitmap_info.offset) {
1065 /* No permanent bitmap - fill with '1s'. */
1066 store->filemap = NULL;
1067 store->file_pages = 0;
1068 for (i = 0; i < chunks ; i++) {
1069 /* if the disk bit is set, set the memory bit */
1070 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1072 md_bitmap_set_memory_bits(bitmap,
1073 (sector_t)i << bitmap->counts.chunkshift,
1079 outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1081 pr_warn("%s: bitmap file is out of date, doing full recovery\n", bmname(bitmap));
1083 if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1084 pr_warn("%s: bitmap file too short %lu < %lu\n",
1086 (unsigned long) i_size_read(file->f_mapping->host),
1093 if (!bitmap->mddev->bitmap_info.external)
1094 offset = sizeof(bitmap_super_t);
1096 if (mddev_is_clustered(bitmap->mddev))
1097 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1099 for (i = 0; i < chunks; i++) {
1101 index = file_page_index(&bitmap->storage, i);
1102 bit = file_page_offset(&bitmap->storage, i);
1103 if (index != oldindex) { /* this is a new page, read it in */
1105 /* unmap the old page, we're done with it */
1106 if (index == store->file_pages-1)
1107 count = store->bytes - index * PAGE_SIZE;
1110 page = store->filemap[index];
1112 ret = read_page(file, index, bitmap,
1117 bitmap->mddev->bitmap_info.offset,
1119 index + node_offset, count);
1128 * if bitmap is out of date, dirty the
1129 * whole page and write it out
1131 paddr = kmap_atomic(page);
1132 memset(paddr + offset, 0xff,
1133 PAGE_SIZE - offset);
1134 kunmap_atomic(paddr);
1135 write_page(bitmap, page, 1);
1138 if (test_bit(BITMAP_WRITE_ERROR,
1143 paddr = kmap_atomic(page);
1144 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1145 b = test_bit(bit, paddr);
1147 b = test_bit_le(bit, paddr);
1148 kunmap_atomic(paddr);
1150 /* if the disk bit is set, set the memory bit */
1151 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1153 md_bitmap_set_memory_bits(bitmap,
1154 (sector_t)i << bitmap->counts.chunkshift,
1161 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
1162 bmname(bitmap), store->file_pages,
1168 pr_warn("%s: bitmap initialisation failed: %d\n",
1169 bmname(bitmap), ret);
1173 void md_bitmap_write_all(struct bitmap *bitmap)
1175 /* We don't actually write all bitmap blocks here,
1176 * just flag them as needing to be written
1180 if (!bitmap || !bitmap->storage.filemap)
1182 if (bitmap->storage.file)
1183 /* Only one copy, so nothing needed */
1186 for (i = 0; i < bitmap->storage.file_pages; i++)
1187 set_page_attr(bitmap, i,
1188 BITMAP_PAGE_NEEDWRITE);
1189 bitmap->allclean = 0;
1192 static void md_bitmap_count_page(struct bitmap_counts *bitmap,
1193 sector_t offset, int inc)
1195 sector_t chunk = offset >> bitmap->chunkshift;
1196 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1197 bitmap->bp[page].count += inc;
1198 md_bitmap_checkfree(bitmap, page);
1201 static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1203 sector_t chunk = offset >> bitmap->chunkshift;
1204 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1205 struct bitmap_page *bp = &bitmap->bp[page];
1211 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1212 sector_t offset, sector_t *blocks,
1216 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1220 void md_bitmap_daemon_work(struct mddev *mddev)
1222 struct bitmap *bitmap;
1224 unsigned long nextpage;
1226 struct bitmap_counts *counts;
1228 /* Use a mutex to guard daemon_work against
1231 mutex_lock(&mddev->bitmap_info.mutex);
1232 bitmap = mddev->bitmap;
1233 if (bitmap == NULL) {
1234 mutex_unlock(&mddev->bitmap_info.mutex);
1237 if (time_before(jiffies, bitmap->daemon_lastrun
1238 + mddev->bitmap_info.daemon_sleep))
1241 bitmap->daemon_lastrun = jiffies;
1242 if (bitmap->allclean) {
1243 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1246 bitmap->allclean = 1;
1248 if (bitmap->mddev->queue)
1249 blk_add_trace_msg(bitmap->mddev->queue,
1250 "md bitmap_daemon_work");
1252 /* Any file-page which is PENDING now needs to be written.
1253 * So set NEEDWRITE now, then after we make any last-minute changes
1256 for (j = 0; j < bitmap->storage.file_pages; j++)
1257 if (test_and_clear_page_attr(bitmap, j,
1258 BITMAP_PAGE_PENDING))
1259 set_page_attr(bitmap, j,
1260 BITMAP_PAGE_NEEDWRITE);
1262 if (bitmap->need_sync &&
1263 mddev->bitmap_info.external == 0) {
1264 /* Arrange for superblock update as well as
1267 bitmap->need_sync = 0;
1268 if (bitmap->storage.filemap) {
1269 sb = kmap_atomic(bitmap->storage.sb_page);
1270 sb->events_cleared =
1271 cpu_to_le64(bitmap->events_cleared);
1273 set_page_attr(bitmap, 0,
1274 BITMAP_PAGE_NEEDWRITE);
1277 /* Now look at the bitmap counters and if any are '2' or '1',
1278 * decrement and handle accordingly.
1280 counts = &bitmap->counts;
1281 spin_lock_irq(&counts->lock);
1283 for (j = 0; j < counts->chunks; j++) {
1284 bitmap_counter_t *bmc;
1285 sector_t block = (sector_t)j << counts->chunkshift;
1287 if (j == nextpage) {
1288 nextpage += PAGE_COUNTER_RATIO;
1289 if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1290 j |= PAGE_COUNTER_MASK;
1293 counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1296 bmc = md_bitmap_get_counter(counts, block, &blocks, 0);
1298 j |= PAGE_COUNTER_MASK;
1301 if (*bmc == 1 && !bitmap->need_sync) {
1302 /* We can clear the bit */
1304 md_bitmap_count_page(counts, block, -1);
1305 md_bitmap_file_clear_bit(bitmap, block);
1306 } else if (*bmc && *bmc <= 2) {
1308 md_bitmap_set_pending(counts, block);
1309 bitmap->allclean = 0;
1312 spin_unlock_irq(&counts->lock);
1314 md_bitmap_wait_writes(bitmap);
1315 /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1316 * DIRTY pages need to be written by bitmap_unplug so it can wait
1318 * If we find any DIRTY page we stop there and let bitmap_unplug
1319 * handle all the rest. This is important in the case where
1320 * the first blocking holds the superblock and it has been updated.
1321 * We mustn't write any other blocks before the superblock.
1324 j < bitmap->storage.file_pages
1325 && !test_bit(BITMAP_STALE, &bitmap->flags);
1327 if (test_page_attr(bitmap, j,
1329 /* bitmap_unplug will handle the rest */
1331 if (bitmap->storage.filemap &&
1332 test_and_clear_page_attr(bitmap, j,
1333 BITMAP_PAGE_NEEDWRITE)) {
1334 write_page(bitmap, bitmap->storage.filemap[j], 0);
1339 if (bitmap->allclean == 0)
1340 mddev->thread->timeout =
1341 mddev->bitmap_info.daemon_sleep;
1342 mutex_unlock(&mddev->bitmap_info.mutex);
1345 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1346 sector_t offset, sector_t *blocks,
1348 __releases(bitmap->lock)
1349 __acquires(bitmap->lock)
1351 /* If 'create', we might release the lock and reclaim it.
1352 * The lock must have been taken with interrupts enabled.
1353 * If !create, we don't release the lock.
1355 sector_t chunk = offset >> bitmap->chunkshift;
1356 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1357 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1361 if (page >= bitmap->pages) {
1363 * This can happen if bitmap_start_sync goes beyond
1364 * End-of-device while looking for a whole page or
1365 * user set a huge number to sysfs bitmap_set_bits.
1369 err = md_bitmap_checkpage(bitmap, page, create, 0);
1371 if (bitmap->bp[page].hijacked ||
1372 bitmap->bp[page].map == NULL)
1373 csize = ((sector_t)1) << (bitmap->chunkshift +
1374 PAGE_COUNTER_SHIFT);
1376 csize = ((sector_t)1) << bitmap->chunkshift;
1377 *blocks = csize - (offset & (csize - 1));
1382 /* now locked ... */
1384 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1385 /* should we use the first or second counter field
1386 * of the hijacked pointer? */
1387 int hi = (pageoff > PAGE_COUNTER_MASK);
1388 return &((bitmap_counter_t *)
1389 &bitmap->bp[page].map)[hi];
1390 } else /* page is allocated */
1391 return (bitmap_counter_t *)
1392 &(bitmap->bp[page].map[pageoff]);
1395 int md_bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1402 atomic_inc(&bitmap->behind_writes);
1403 bw = atomic_read(&bitmap->behind_writes);
1404 if (bw > bitmap->behind_writes_used)
1405 bitmap->behind_writes_used = bw;
1407 pr_debug("inc write-behind count %d/%lu\n",
1408 bw, bitmap->mddev->bitmap_info.max_write_behind);
1413 bitmap_counter_t *bmc;
1415 spin_lock_irq(&bitmap->counts.lock);
1416 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1418 spin_unlock_irq(&bitmap->counts.lock);
1422 if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1423 DEFINE_WAIT(__wait);
1424 /* note that it is safe to do the prepare_to_wait
1425 * after the test as long as we do it before dropping
1428 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1429 TASK_UNINTERRUPTIBLE);
1430 spin_unlock_irq(&bitmap->counts.lock);
1432 finish_wait(&bitmap->overflow_wait, &__wait);
1438 md_bitmap_file_set_bit(bitmap, offset);
1439 md_bitmap_count_page(&bitmap->counts, offset, 1);
1447 spin_unlock_irq(&bitmap->counts.lock);
1450 if (sectors > blocks)
1457 EXPORT_SYMBOL(md_bitmap_startwrite);
1459 void md_bitmap_endwrite(struct bitmap *bitmap, sector_t offset,
1460 unsigned long sectors, int success, int behind)
1465 if (atomic_dec_and_test(&bitmap->behind_writes))
1466 wake_up(&bitmap->behind_wait);
1467 pr_debug("dec write-behind count %d/%lu\n",
1468 atomic_read(&bitmap->behind_writes),
1469 bitmap->mddev->bitmap_info.max_write_behind);
1474 unsigned long flags;
1475 bitmap_counter_t *bmc;
1477 spin_lock_irqsave(&bitmap->counts.lock, flags);
1478 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1480 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1484 if (success && !bitmap->mddev->degraded &&
1485 bitmap->events_cleared < bitmap->mddev->events) {
1486 bitmap->events_cleared = bitmap->mddev->events;
1487 bitmap->need_sync = 1;
1488 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1491 if (!success && !NEEDED(*bmc))
1492 *bmc |= NEEDED_MASK;
1494 if (COUNTER(*bmc) == COUNTER_MAX)
1495 wake_up(&bitmap->overflow_wait);
1499 md_bitmap_set_pending(&bitmap->counts, offset);
1500 bitmap->allclean = 0;
1502 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1504 if (sectors > blocks)
1510 EXPORT_SYMBOL(md_bitmap_endwrite);
1512 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1515 bitmap_counter_t *bmc;
1517 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1519 return 1; /* always resync if no bitmap */
1521 spin_lock_irq(&bitmap->counts.lock);
1522 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1528 else if (NEEDED(*bmc)) {
1530 if (!degraded) { /* don't set/clear bits if degraded */
1531 *bmc |= RESYNC_MASK;
1532 *bmc &= ~NEEDED_MASK;
1536 spin_unlock_irq(&bitmap->counts.lock);
1540 int md_bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1543 /* bitmap_start_sync must always report on multiples of whole
1544 * pages, otherwise resync (which is very PAGE_SIZE based) will
1546 * So call __bitmap_start_sync repeatedly (if needed) until
1547 * At least PAGE_SIZE>>9 blocks are covered.
1548 * Return the 'or' of the result.
1554 while (*blocks < (PAGE_SIZE>>9)) {
1555 rv |= __bitmap_start_sync(bitmap, offset,
1556 &blocks1, degraded);
1562 EXPORT_SYMBOL(md_bitmap_start_sync);
1564 void md_bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1566 bitmap_counter_t *bmc;
1567 unsigned long flags;
1569 if (bitmap == NULL) {
1573 spin_lock_irqsave(&bitmap->counts.lock, flags);
1574 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1579 *bmc &= ~RESYNC_MASK;
1581 if (!NEEDED(*bmc) && aborted)
1582 *bmc |= NEEDED_MASK;
1585 md_bitmap_set_pending(&bitmap->counts, offset);
1586 bitmap->allclean = 0;
1591 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1593 EXPORT_SYMBOL(md_bitmap_end_sync);
1595 void md_bitmap_close_sync(struct bitmap *bitmap)
1597 /* Sync has finished, and any bitmap chunks that weren't synced
1598 * properly have been aborted. It remains to us to clear the
1599 * RESYNC bit wherever it is still on
1601 sector_t sector = 0;
1605 while (sector < bitmap->mddev->resync_max_sectors) {
1606 md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1610 EXPORT_SYMBOL(md_bitmap_close_sync);
1612 void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
1620 bitmap->last_end_sync = jiffies;
1623 if (!force && time_before(jiffies, (bitmap->last_end_sync
1624 + bitmap->mddev->bitmap_info.daemon_sleep)))
1626 wait_event(bitmap->mddev->recovery_wait,
1627 atomic_read(&bitmap->mddev->recovery_active) == 0);
1629 bitmap->mddev->curr_resync_completed = sector;
1630 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
1631 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1633 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1634 md_bitmap_end_sync(bitmap, s, &blocks, 0);
1637 bitmap->last_end_sync = jiffies;
1638 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed);
1640 EXPORT_SYMBOL(md_bitmap_cond_end_sync);
1642 void md_bitmap_sync_with_cluster(struct mddev *mddev,
1643 sector_t old_lo, sector_t old_hi,
1644 sector_t new_lo, sector_t new_hi)
1646 struct bitmap *bitmap = mddev->bitmap;
1647 sector_t sector, blocks = 0;
1649 for (sector = old_lo; sector < new_lo; ) {
1650 md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1653 WARN((blocks > new_lo) && old_lo, "alignment is not correct for lo\n");
1655 for (sector = old_hi; sector < new_hi; ) {
1656 md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1659 WARN((blocks > new_hi) && old_hi, "alignment is not correct for hi\n");
1661 EXPORT_SYMBOL(md_bitmap_sync_with_cluster);
1663 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1665 /* For each chunk covered by any of these sectors, set the
1666 * counter to 2 and possibly set resync_needed. They should all
1667 * be 0 at this point
1671 bitmap_counter_t *bmc;
1672 spin_lock_irq(&bitmap->counts.lock);
1673 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1675 spin_unlock_irq(&bitmap->counts.lock);
1680 md_bitmap_count_page(&bitmap->counts, offset, 1);
1681 md_bitmap_set_pending(&bitmap->counts, offset);
1682 bitmap->allclean = 0;
1685 *bmc |= NEEDED_MASK;
1686 spin_unlock_irq(&bitmap->counts.lock);
1689 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
1690 void md_bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1692 unsigned long chunk;
1694 for (chunk = s; chunk <= e; chunk++) {
1695 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1696 md_bitmap_set_memory_bits(bitmap, sec, 1);
1697 md_bitmap_file_set_bit(bitmap, sec);
1698 if (sec < bitmap->mddev->recovery_cp)
1699 /* We are asserting that the array is dirty,
1700 * so move the recovery_cp address back so
1701 * that it is obvious that it is dirty
1703 bitmap->mddev->recovery_cp = sec;
1708 * flush out any pending updates
1710 void md_bitmap_flush(struct mddev *mddev)
1712 struct bitmap *bitmap = mddev->bitmap;
1715 if (!bitmap) /* there was no bitmap */
1718 /* run the daemon_work three time to ensure everything is flushed
1721 sleep = mddev->bitmap_info.daemon_sleep * 2;
1722 bitmap->daemon_lastrun -= sleep;
1723 md_bitmap_daemon_work(mddev);
1724 bitmap->daemon_lastrun -= sleep;
1725 md_bitmap_daemon_work(mddev);
1726 bitmap->daemon_lastrun -= sleep;
1727 md_bitmap_daemon_work(mddev);
1728 if (mddev->bitmap_info.external)
1729 md_super_wait(mddev);
1730 md_bitmap_update_sb(bitmap);
1734 * free memory that was allocated
1736 void md_bitmap_free(struct bitmap *bitmap)
1738 unsigned long k, pages;
1739 struct bitmap_page *bp;
1741 if (!bitmap) /* there was no bitmap */
1744 if (bitmap->sysfs_can_clear)
1745 sysfs_put(bitmap->sysfs_can_clear);
1747 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
1748 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1749 md_cluster_stop(bitmap->mddev);
1751 /* Shouldn't be needed - but just in case.... */
1752 wait_event(bitmap->write_wait,
1753 atomic_read(&bitmap->pending_writes) == 0);
1755 /* release the bitmap file */
1756 md_bitmap_file_unmap(&bitmap->storage);
1758 bp = bitmap->counts.bp;
1759 pages = bitmap->counts.pages;
1761 /* free all allocated memory */
1763 if (bp) /* deallocate the page memory */
1764 for (k = 0; k < pages; k++)
1765 if (bp[k].map && !bp[k].hijacked)
1770 EXPORT_SYMBOL(md_bitmap_free);
1772 void md_bitmap_wait_behind_writes(struct mddev *mddev)
1774 struct bitmap *bitmap = mddev->bitmap;
1776 /* wait for behind writes to complete */
1777 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
1778 pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
1780 /* need to kick something here to make sure I/O goes? */
1781 wait_event(bitmap->behind_wait,
1782 atomic_read(&bitmap->behind_writes) == 0);
1786 void md_bitmap_destroy(struct mddev *mddev)
1788 struct bitmap *bitmap = mddev->bitmap;
1790 if (!bitmap) /* there was no bitmap */
1793 md_bitmap_wait_behind_writes(mddev);
1794 if (!mddev->serialize_policy)
1795 mddev_destroy_serial_pool(mddev, NULL, true);
1797 mutex_lock(&mddev->bitmap_info.mutex);
1798 spin_lock(&mddev->lock);
1799 mddev->bitmap = NULL; /* disconnect from the md device */
1800 spin_unlock(&mddev->lock);
1801 mutex_unlock(&mddev->bitmap_info.mutex);
1803 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1805 md_bitmap_free(bitmap);
1809 * initialize the bitmap structure
1810 * if this returns an error, bitmap_destroy must be called to do clean up
1811 * once mddev->bitmap is set
1813 struct bitmap *md_bitmap_create(struct mddev *mddev, int slot)
1815 struct bitmap *bitmap;
1816 sector_t blocks = mddev->resync_max_sectors;
1817 struct file *file = mddev->bitmap_info.file;
1819 struct kernfs_node *bm = NULL;
1821 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1823 BUG_ON(file && mddev->bitmap_info.offset);
1825 if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
1826 pr_notice("md/raid:%s: array with journal cannot have bitmap\n",
1828 return ERR_PTR(-EBUSY);
1831 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1833 return ERR_PTR(-ENOMEM);
1835 spin_lock_init(&bitmap->counts.lock);
1836 atomic_set(&bitmap->pending_writes, 0);
1837 init_waitqueue_head(&bitmap->write_wait);
1838 init_waitqueue_head(&bitmap->overflow_wait);
1839 init_waitqueue_head(&bitmap->behind_wait);
1841 bitmap->mddev = mddev;
1842 bitmap->cluster_slot = slot;
1845 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1847 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1850 bitmap->sysfs_can_clear = NULL;
1852 bitmap->storage.file = file;
1855 /* As future accesses to this file will use bmap,
1856 * and bypass the page cache, we must sync the file
1861 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1862 if (!mddev->bitmap_info.external) {
1864 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1865 * instructing us to create a new on-disk bitmap instance.
1867 if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1868 err = md_bitmap_new_disk_sb(bitmap);
1870 err = md_bitmap_read_sb(bitmap);
1873 if (mddev->bitmap_info.chunksize == 0 ||
1874 mddev->bitmap_info.daemon_sleep == 0)
1875 /* chunksize and time_base need to be
1882 bitmap->daemon_lastrun = jiffies;
1883 err = md_bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1887 pr_debug("created bitmap (%lu pages) for device %s\n",
1888 bitmap->counts.pages, bmname(bitmap));
1890 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1896 md_bitmap_free(bitmap);
1897 return ERR_PTR(err);
1900 int md_bitmap_load(struct mddev *mddev)
1904 sector_t sector = 0;
1905 struct bitmap *bitmap = mddev->bitmap;
1906 struct md_rdev *rdev;
1911 rdev_for_each(rdev, mddev)
1912 mddev_create_serial_pool(mddev, rdev, true);
1914 if (mddev_is_clustered(mddev))
1915 md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
1917 /* Clear out old bitmap info first: Either there is none, or we
1918 * are resuming after someone else has possibly changed things,
1919 * so we should forget old cached info.
1920 * All chunks should be clean, but some might need_sync.
1922 while (sector < mddev->resync_max_sectors) {
1924 md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1927 md_bitmap_close_sync(bitmap);
1929 if (mddev->degraded == 0
1930 || bitmap->events_cleared == mddev->events)
1931 /* no need to keep dirty bits to optimise a
1932 * re-add of a missing device */
1933 start = mddev->recovery_cp;
1935 mutex_lock(&mddev->bitmap_info.mutex);
1936 err = md_bitmap_init_from_disk(bitmap, start);
1937 mutex_unlock(&mddev->bitmap_info.mutex);
1941 clear_bit(BITMAP_STALE, &bitmap->flags);
1943 /* Kick recovery in case any bits were set */
1944 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1946 mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1947 md_wakeup_thread(mddev->thread);
1949 md_bitmap_update_sb(bitmap);
1951 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1956 EXPORT_SYMBOL_GPL(md_bitmap_load);
1958 /* caller need to free returned bitmap with md_bitmap_free() */
1959 struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot)
1962 struct bitmap *bitmap;
1964 bitmap = md_bitmap_create(mddev, slot);
1965 if (IS_ERR(bitmap)) {
1966 rv = PTR_ERR(bitmap);
1970 rv = md_bitmap_init_from_disk(bitmap, 0);
1972 md_bitmap_free(bitmap);
1978 EXPORT_SYMBOL(get_bitmap_from_slot);
1980 /* Loads the bitmap associated with slot and copies the resync information
1983 int md_bitmap_copy_from_slot(struct mddev *mddev, int slot,
1984 sector_t *low, sector_t *high, bool clear_bits)
1987 sector_t block, lo = 0, hi = 0;
1988 struct bitmap_counts *counts;
1989 struct bitmap *bitmap;
1991 bitmap = get_bitmap_from_slot(mddev, slot);
1992 if (IS_ERR(bitmap)) {
1993 pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
1997 counts = &bitmap->counts;
1998 for (j = 0; j < counts->chunks; j++) {
1999 block = (sector_t)j << counts->chunkshift;
2000 if (md_bitmap_file_test_bit(bitmap, block)) {
2004 md_bitmap_file_clear_bit(bitmap, block);
2005 md_bitmap_set_memory_bits(mddev->bitmap, block, 1);
2006 md_bitmap_file_set_bit(mddev->bitmap, block);
2011 md_bitmap_update_sb(bitmap);
2012 /* BITMAP_PAGE_PENDING is set, but bitmap_unplug needs
2013 * BITMAP_PAGE_DIRTY or _NEEDWRITE to write ... */
2014 for (i = 0; i < bitmap->storage.file_pages; i++)
2015 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
2016 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
2017 md_bitmap_unplug(bitmap);
2019 md_bitmap_unplug(mddev->bitmap);
2022 md_bitmap_free(bitmap);
2026 EXPORT_SYMBOL_GPL(md_bitmap_copy_from_slot);
2029 void md_bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
2031 unsigned long chunk_kb;
2032 struct bitmap_counts *counts;
2037 counts = &bitmap->counts;
2039 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
2040 seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
2042 counts->pages - counts->missing_pages,
2044 (counts->pages - counts->missing_pages)
2045 << (PAGE_SHIFT - 10),
2046 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
2047 chunk_kb ? "KB" : "B");
2048 if (bitmap->storage.file) {
2049 seq_printf(seq, ", file: ");
2050 seq_file_path(seq, bitmap->storage.file, " \t\n");
2053 seq_printf(seq, "\n");
2056 int md_bitmap_resize(struct bitmap *bitmap, sector_t blocks,
2057 int chunksize, int init)
2059 /* If chunk_size is 0, choose an appropriate chunk size.
2060 * Then possibly allocate new storage space.
2061 * Then quiesce, copy bits, replace bitmap, and re-start
2063 * This function is called both to set up the initial bitmap
2064 * and to resize the bitmap while the array is active.
2065 * If this happens as a result of the array being resized,
2066 * chunksize will be zero, and we need to choose a suitable
2067 * chunksize, otherwise we use what we are given.
2069 struct bitmap_storage store;
2070 struct bitmap_counts old_counts;
2071 unsigned long chunks;
2073 sector_t old_blocks, new_blocks;
2077 struct bitmap_page *new_bp;
2079 if (bitmap->storage.file && !init) {
2080 pr_info("md: cannot resize file-based bitmap\n");
2084 if (chunksize == 0) {
2085 /* If there is enough space, leave the chunk size unchanged,
2086 * else increase by factor of two until there is enough space.
2089 long space = bitmap->mddev->bitmap_info.space;
2092 /* We don't know how much space there is, so limit
2093 * to current size - in sectors.
2095 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
2096 if (!bitmap->mddev->bitmap_info.external)
2097 bytes += sizeof(bitmap_super_t);
2098 space = DIV_ROUND_UP(bytes, 512);
2099 bitmap->mddev->bitmap_info.space = space;
2101 chunkshift = bitmap->counts.chunkshift;
2104 /* 'chunkshift' is shift from block size to chunk size */
2106 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2107 bytes = DIV_ROUND_UP(chunks, 8);
2108 if (!bitmap->mddev->bitmap_info.external)
2109 bytes += sizeof(bitmap_super_t);
2110 } while (bytes > (space << 9) && (chunkshift + BITMAP_BLOCK_SHIFT) <
2111 (BITS_PER_BYTE * sizeof(((bitmap_super_t *)0)->chunksize) - 1));
2113 chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
2115 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2116 memset(&store, 0, sizeof(store));
2117 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
2118 ret = md_bitmap_storage_alloc(&store, chunks,
2119 !bitmap->mddev->bitmap_info.external,
2120 mddev_is_clustered(bitmap->mddev)
2121 ? bitmap->cluster_slot : 0);
2123 md_bitmap_file_unmap(&store);
2127 pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
2129 new_bp = kcalloc(pages, sizeof(*new_bp), GFP_KERNEL);
2132 md_bitmap_file_unmap(&store);
2137 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2139 store.file = bitmap->storage.file;
2140 bitmap->storage.file = NULL;
2142 if (store.sb_page && bitmap->storage.sb_page)
2143 memcpy(page_address(store.sb_page),
2144 page_address(bitmap->storage.sb_page),
2145 sizeof(bitmap_super_t));
2146 spin_lock_irq(&bitmap->counts.lock);
2147 md_bitmap_file_unmap(&bitmap->storage);
2148 bitmap->storage = store;
2150 old_counts = bitmap->counts;
2151 bitmap->counts.bp = new_bp;
2152 bitmap->counts.pages = pages;
2153 bitmap->counts.missing_pages = pages;
2154 bitmap->counts.chunkshift = chunkshift;
2155 bitmap->counts.chunks = chunks;
2156 bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift +
2157 BITMAP_BLOCK_SHIFT);
2159 blocks = min(old_counts.chunks << old_counts.chunkshift,
2160 chunks << chunkshift);
2162 /* For cluster raid, need to pre-allocate bitmap */
2163 if (mddev_is_clustered(bitmap->mddev)) {
2165 for (page = 0; page < pages; page++) {
2166 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1);
2170 /* deallocate the page memory */
2171 for (k = 0; k < page; k++) {
2172 kfree(new_bp[k].map);
2176 /* restore some fields from old_counts */
2177 bitmap->counts.bp = old_counts.bp;
2178 bitmap->counts.pages = old_counts.pages;
2179 bitmap->counts.missing_pages = old_counts.pages;
2180 bitmap->counts.chunkshift = old_counts.chunkshift;
2181 bitmap->counts.chunks = old_counts.chunks;
2182 bitmap->mddev->bitmap_info.chunksize =
2183 1UL << (old_counts.chunkshift + BITMAP_BLOCK_SHIFT);
2184 blocks = old_counts.chunks << old_counts.chunkshift;
2185 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2188 bitmap->counts.bp[page].count += 1;
2192 for (block = 0; block < blocks; ) {
2193 bitmap_counter_t *bmc_old, *bmc_new;
2196 bmc_old = md_bitmap_get_counter(&old_counts, block, &old_blocks, 0);
2197 set = bmc_old && NEEDED(*bmc_old);
2200 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2202 if (*bmc_new == 0) {
2203 /* need to set on-disk bits too. */
2204 sector_t end = block + new_blocks;
2205 sector_t start = block >> chunkshift;
2207 start <<= chunkshift;
2208 while (start < end) {
2209 md_bitmap_file_set_bit(bitmap, block);
2210 start += 1 << chunkshift;
2213 md_bitmap_count_page(&bitmap->counts, block, 1);
2214 md_bitmap_set_pending(&bitmap->counts, block);
2216 *bmc_new |= NEEDED_MASK;
2218 if (new_blocks < old_blocks)
2219 old_blocks = new_blocks;
2221 block += old_blocks;
2224 if (bitmap->counts.bp != old_counts.bp) {
2226 for (k = 0; k < old_counts.pages; k++)
2227 if (!old_counts.bp[k].hijacked)
2228 kfree(old_counts.bp[k].map);
2229 kfree(old_counts.bp);
2234 while (block < (chunks << chunkshift)) {
2235 bitmap_counter_t *bmc;
2236 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2238 /* new space. It needs to be resynced, so
2239 * we set NEEDED_MASK.
2242 *bmc = NEEDED_MASK | 2;
2243 md_bitmap_count_page(&bitmap->counts, block, 1);
2244 md_bitmap_set_pending(&bitmap->counts, block);
2247 block += new_blocks;
2249 for (i = 0; i < bitmap->storage.file_pages; i++)
2250 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2252 spin_unlock_irq(&bitmap->counts.lock);
2255 md_bitmap_unplug(bitmap);
2256 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2262 EXPORT_SYMBOL_GPL(md_bitmap_resize);
2265 location_show(struct mddev *mddev, char *page)
2268 if (mddev->bitmap_info.file)
2269 len = sprintf(page, "file");
2270 else if (mddev->bitmap_info.offset)
2271 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
2273 len = sprintf(page, "none");
2274 len += sprintf(page+len, "\n");
2279 location_store(struct mddev *mddev, const char *buf, size_t len)
2283 rv = mddev_lock(mddev);
2287 if (!mddev->pers->quiesce) {
2291 if (mddev->recovery || mddev->sync_thread) {
2297 if (mddev->bitmap || mddev->bitmap_info.file ||
2298 mddev->bitmap_info.offset) {
2299 /* bitmap already configured. Only option is to clear it */
2300 if (strncmp(buf, "none", 4) != 0) {
2305 mddev_suspend(mddev);
2306 md_bitmap_destroy(mddev);
2307 mddev_resume(mddev);
2309 mddev->bitmap_info.offset = 0;
2310 if (mddev->bitmap_info.file) {
2311 struct file *f = mddev->bitmap_info.file;
2312 mddev->bitmap_info.file = NULL;
2316 /* No bitmap, OK to set a location */
2318 if (strncmp(buf, "none", 4) == 0)
2319 /* nothing to be done */;
2320 else if (strncmp(buf, "file:", 5) == 0) {
2321 /* Not supported yet */
2326 rv = kstrtoll(buf+1, 10, &offset);
2328 rv = kstrtoll(buf, 10, &offset);
2335 if (mddev->bitmap_info.external == 0 &&
2336 mddev->major_version == 0 &&
2337 offset != mddev->bitmap_info.default_offset) {
2341 mddev->bitmap_info.offset = offset;
2343 struct bitmap *bitmap;
2344 bitmap = md_bitmap_create(mddev, -1);
2345 mddev_suspend(mddev);
2347 rv = PTR_ERR(bitmap);
2349 mddev->bitmap = bitmap;
2350 rv = md_bitmap_load(mddev);
2352 mddev->bitmap_info.offset = 0;
2355 md_bitmap_destroy(mddev);
2356 mddev_resume(mddev);
2359 mddev_resume(mddev);
2363 if (!mddev->external) {
2364 /* Ensure new bitmap info is stored in
2365 * metadata promptly.
2367 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2368 md_wakeup_thread(mddev->thread);
2372 mddev_unlock(mddev);
2378 static struct md_sysfs_entry bitmap_location =
2379 __ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2381 /* 'bitmap/space' is the space available at 'location' for the
2382 * bitmap. This allows the kernel to know when it is safe to
2383 * resize the bitmap to match a resized array.
2386 space_show(struct mddev *mddev, char *page)
2388 return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2392 space_store(struct mddev *mddev, const char *buf, size_t len)
2394 unsigned long sectors;
2397 rv = kstrtoul(buf, 10, §ors);
2404 if (mddev->bitmap &&
2405 sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2406 return -EFBIG; /* Bitmap is too big for this small space */
2408 /* could make sure it isn't too big, but that isn't really
2409 * needed - user-space should be careful.
2411 mddev->bitmap_info.space = sectors;
2415 static struct md_sysfs_entry bitmap_space =
2416 __ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2419 timeout_show(struct mddev *mddev, char *page)
2422 unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2423 unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2425 len = sprintf(page, "%lu", secs);
2427 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2428 len += sprintf(page+len, "\n");
2433 timeout_store(struct mddev *mddev, const char *buf, size_t len)
2435 /* timeout can be set at any time */
2436 unsigned long timeout;
2437 int rv = strict_strtoul_scaled(buf, &timeout, 4);
2441 /* just to make sure we don't overflow... */
2442 if (timeout >= LONG_MAX / HZ)
2445 timeout = timeout * HZ / 10000;
2447 if (timeout >= MAX_SCHEDULE_TIMEOUT)
2448 timeout = MAX_SCHEDULE_TIMEOUT-1;
2451 mddev->bitmap_info.daemon_sleep = timeout;
2452 if (mddev->thread) {
2453 /* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
2454 * the bitmap is all clean and we don't need to
2455 * adjust the timeout right now
2457 if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
2458 mddev->thread->timeout = timeout;
2459 md_wakeup_thread(mddev->thread);
2465 static struct md_sysfs_entry bitmap_timeout =
2466 __ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2469 backlog_show(struct mddev *mddev, char *page)
2471 return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2475 backlog_store(struct mddev *mddev, const char *buf, size_t len)
2477 unsigned long backlog;
2478 unsigned long old_mwb = mddev->bitmap_info.max_write_behind;
2479 struct md_rdev *rdev;
2480 bool has_write_mostly = false;
2481 int rv = kstrtoul(buf, 10, &backlog);
2484 if (backlog > COUNTER_MAX)
2487 rv = mddev_lock(mddev);
2492 * Without write mostly device, it doesn't make sense to set
2493 * backlog for max_write_behind.
2495 rdev_for_each(rdev, mddev) {
2496 if (test_bit(WriteMostly, &rdev->flags)) {
2497 has_write_mostly = true;
2501 if (!has_write_mostly) {
2502 pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n",
2504 mddev_unlock(mddev);
2508 mddev->bitmap_info.max_write_behind = backlog;
2509 if (!backlog && mddev->serial_info_pool) {
2510 /* serial_info_pool is not needed if backlog is zero */
2511 if (!mddev->serialize_policy)
2512 mddev_destroy_serial_pool(mddev, NULL, false);
2513 } else if (backlog && !mddev->serial_info_pool) {
2514 /* serial_info_pool is needed since backlog is not zero */
2515 rdev_for_each(rdev, mddev)
2516 mddev_create_serial_pool(mddev, rdev, false);
2518 if (old_mwb != backlog)
2519 md_bitmap_update_sb(mddev->bitmap);
2521 mddev_unlock(mddev);
2525 static struct md_sysfs_entry bitmap_backlog =
2526 __ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2529 chunksize_show(struct mddev *mddev, char *page)
2531 return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2535 chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2537 /* Can only be changed when no bitmap is active */
2539 unsigned long csize;
2542 rv = kstrtoul(buf, 10, &csize);
2546 !is_power_of_2(csize))
2548 if (BITS_PER_LONG > 32 && csize >= (1ULL << (BITS_PER_BYTE *
2549 sizeof(((bitmap_super_t *)0)->chunksize))))
2551 mddev->bitmap_info.chunksize = csize;
2555 static struct md_sysfs_entry bitmap_chunksize =
2556 __ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2558 static ssize_t metadata_show(struct mddev *mddev, char *page)
2560 if (mddev_is_clustered(mddev))
2561 return sprintf(page, "clustered\n");
2562 return sprintf(page, "%s\n", (mddev->bitmap_info.external
2563 ? "external" : "internal"));
2566 static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2568 if (mddev->bitmap ||
2569 mddev->bitmap_info.file ||
2570 mddev->bitmap_info.offset)
2572 if (strncmp(buf, "external", 8) == 0)
2573 mddev->bitmap_info.external = 1;
2574 else if ((strncmp(buf, "internal", 8) == 0) ||
2575 (strncmp(buf, "clustered", 9) == 0))
2576 mddev->bitmap_info.external = 0;
2582 static struct md_sysfs_entry bitmap_metadata =
2583 __ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2585 static ssize_t can_clear_show(struct mddev *mddev, char *page)
2588 spin_lock(&mddev->lock);
2590 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2593 len = sprintf(page, "\n");
2594 spin_unlock(&mddev->lock);
2598 static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2600 if (mddev->bitmap == NULL)
2602 if (strncmp(buf, "false", 5) == 0)
2603 mddev->bitmap->need_sync = 1;
2604 else if (strncmp(buf, "true", 4) == 0) {
2605 if (mddev->degraded)
2607 mddev->bitmap->need_sync = 0;
2613 static struct md_sysfs_entry bitmap_can_clear =
2614 __ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2617 behind_writes_used_show(struct mddev *mddev, char *page)
2620 spin_lock(&mddev->lock);
2621 if (mddev->bitmap == NULL)
2622 ret = sprintf(page, "0\n");
2624 ret = sprintf(page, "%lu\n",
2625 mddev->bitmap->behind_writes_used);
2626 spin_unlock(&mddev->lock);
2631 behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2634 mddev->bitmap->behind_writes_used = 0;
2638 static struct md_sysfs_entry max_backlog_used =
2639 __ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2640 behind_writes_used_show, behind_writes_used_reset);
2642 static struct attribute *md_bitmap_attrs[] = {
2643 &bitmap_location.attr,
2645 &bitmap_timeout.attr,
2646 &bitmap_backlog.attr,
2647 &bitmap_chunksize.attr,
2648 &bitmap_metadata.attr,
2649 &bitmap_can_clear.attr,
2650 &max_backlog_used.attr,
2653 struct attribute_group md_bitmap_group = {
2655 .attrs = md_bitmap_attrs,