1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2018 HUAWEI, Inc.
4 * https://www.huawei.com/
8 #include <linux/prefetch.h>
10 #include <trace/events/erofs.h>
13 * since pclustersize is variable for big pcluster feature, introduce slab
14 * pools implementation for different pcluster sizes.
16 struct z_erofs_pcluster_slab {
17 struct kmem_cache *slab;
18 unsigned int maxpages;
22 #define _PCLP(n) { .maxpages = n }
24 static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = {
25 _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128),
26 _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES)
29 static void z_erofs_destroy_pcluster_pool(void)
33 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
34 if (!pcluster_pool[i].slab)
36 kmem_cache_destroy(pcluster_pool[i].slab);
37 pcluster_pool[i].slab = NULL;
41 static int z_erofs_create_pcluster_pool(void)
43 struct z_erofs_pcluster_slab *pcs;
44 struct z_erofs_pcluster *a;
47 for (pcs = pcluster_pool;
48 pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) {
49 size = struct_size(a, compressed_pages, pcs->maxpages);
51 sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages);
52 pcs->slab = kmem_cache_create(pcs->name, size, 0,
53 SLAB_RECLAIM_ACCOUNT, NULL);
57 z_erofs_destroy_pcluster_pool();
63 static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages)
67 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
68 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i;
69 struct z_erofs_pcluster *pcl;
71 if (nrpages > pcs->maxpages)
74 pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS);
76 return ERR_PTR(-ENOMEM);
77 pcl->pclusterpages = nrpages;
80 return ERR_PTR(-EINVAL);
83 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl)
85 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
88 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
89 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i;
91 if (pclusterpages > pcs->maxpages)
94 kmem_cache_free(pcs->slab, pcl);
100 /* how to allocate cached pages for a pcluster */
101 enum z_erofs_cache_alloctype {
102 DONTALLOC, /* don't allocate any cached pages */
104 * try to use cached I/O if page allocation succeeds or fallback
105 * to in-place I/O instead to avoid any direct reclaim.
111 * tagged pointer with 1-bit tag for all compressed pages
112 * tag 0 - the page is just found with an extra page reference
114 typedef tagptr1_t compressed_page_t;
116 #define tag_compressed_page_justfound(page) \
117 tagptr_fold(compressed_page_t, page, 1)
119 static struct workqueue_struct *z_erofs_workqueue __read_mostly;
121 void z_erofs_exit_zip_subsystem(void)
123 destroy_workqueue(z_erofs_workqueue);
124 z_erofs_destroy_pcluster_pool();
127 static inline int z_erofs_init_workqueue(void)
129 const unsigned int onlinecpus = num_possible_cpus();
132 * no need to spawn too many threads, limiting threads could minimum
133 * scheduling overhead, perhaps per-CPU threads should be better?
135 z_erofs_workqueue = alloc_workqueue("erofs_unzipd",
136 WQ_UNBOUND | WQ_HIGHPRI,
137 onlinecpus + onlinecpus / 4);
138 return z_erofs_workqueue ? 0 : -ENOMEM;
141 int __init z_erofs_init_zip_subsystem(void)
143 int err = z_erofs_create_pcluster_pool();
147 err = z_erofs_init_workqueue();
149 z_erofs_destroy_pcluster_pool();
153 enum z_erofs_collectmode {
157 * The current collection was the tail of an exist chain, in addition
158 * that the previous processed chained collections are all decided to
159 * be hooked up to it.
160 * A new chain will be created for the remaining collections which are
161 * not processed yet, therefore different from COLLECT_PRIMARY_FOLLOWED,
162 * the next collection cannot reuse the whole page safely in
163 * the following scenario:
164 * ________________________________________________________________
165 * | tail (partial) page | head (partial) page |
166 * | (belongs to the next cl) | (belongs to the current cl) |
167 * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________|
169 COLLECT_PRIMARY_HOOKED,
171 * a weak form of COLLECT_PRIMARY_FOLLOWED, the difference is that it
172 * could be dispatched into bypass queue later due to uptodated managed
173 * pages. All related online pages cannot be reused for inplace I/O (or
174 * pagevec) since it can be directly decoded without I/O submission.
176 COLLECT_PRIMARY_FOLLOWED_NOINPLACE,
178 * The current collection has been linked with the owned chain, and
179 * could also be linked with the remaining collections, which means
180 * if the processing page is the tail page of the collection, thus
181 * the current collection can safely use the whole page (since
182 * the previous collection is under control) for in-place I/O, as
184 * ________________________________________________________________
185 * | tail (partial) page | head (partial) page |
186 * | (of the current cl) | (of the previous collection) |
187 * | PRIMARY_FOLLOWED or | |
188 * |_____PRIMARY_HOOKED___|____________PRIMARY_FOLLOWED____________|
190 * [ (*) the above page can be used as inplace I/O. ]
192 COLLECT_PRIMARY_FOLLOWED,
195 struct z_erofs_decompress_frontend {
196 struct inode *const inode;
197 struct erofs_map_blocks map;
199 struct z_erofs_pagevec_ctor vector;
201 struct z_erofs_pcluster *pcl, *tailpcl;
202 /* a pointer used to pick up inplace I/O pages */
203 struct page **icpage_ptr;
204 z_erofs_next_pcluster_t owned_head;
206 enum z_erofs_collectmode mode;
209 /* used for applying cache strategy on the fly */
211 erofs_off_t headoffset;
214 #define DECOMPRESS_FRONTEND_INIT(__i) { \
215 .inode = __i, .owned_head = Z_EROFS_PCLUSTER_TAIL, \
216 .mode = COLLECT_PRIMARY_FOLLOWED, .backmost = true }
218 static struct page *z_pagemap_global[Z_EROFS_VMAP_GLOBAL_PAGES];
219 static DEFINE_MUTEX(z_pagemap_global_lock);
221 static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe,
222 enum z_erofs_cache_alloctype type,
223 struct page **pagepool)
225 struct address_space *mc = MNGD_MAPPING(EROFS_I_SB(fe->inode));
226 struct z_erofs_pcluster *pcl = fe->pcl;
227 bool standalone = true;
229 * optimistic allocation without direct reclaim since inplace I/O
230 * can be used if low memory otherwise.
232 gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) |
233 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
237 if (fe->mode < COLLECT_PRIMARY_FOLLOWED)
240 pages = pcl->compressed_pages;
241 index = pcl->obj.index;
242 for (; index < pcl->obj.index + pcl->pclusterpages; ++index, ++pages) {
245 struct page *newpage = NULL;
247 /* the compressed page was loaded before */
248 if (READ_ONCE(*pages))
251 page = find_get_page(mc, index);
254 t = tag_compressed_page_justfound(page);
256 /* I/O is needed, no possible to decompress directly */
260 newpage = erofs_allocpage(pagepool, gfp);
263 set_page_private(newpage,
264 Z_EROFS_PREALLOCATED_PAGE);
265 t = tag_compressed_page_justfound(newpage);
267 default: /* DONTALLOC */
272 if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t)))
278 erofs_pagepool_add(pagepool, newpage);
282 * don't do inplace I/O if all compressed pages are available in
283 * managed cache since it can be moved to the bypass queue instead.
286 fe->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;
289 /* called by erofs_shrinker to get rid of all compressed_pages */
290 int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
291 struct erofs_workgroup *grp)
293 struct z_erofs_pcluster *const pcl =
294 container_of(grp, struct z_erofs_pcluster, obj);
297 DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
299 * refcount of workgroup is now freezed as 1,
300 * therefore no need to worry about available decompression users.
302 for (i = 0; i < pcl->pclusterpages; ++i) {
303 struct page *page = pcl->compressed_pages[i];
308 /* block other users from reclaiming or migrating the page */
309 if (!trylock_page(page))
312 if (!erofs_page_is_managed(sbi, page))
315 /* barrier is implied in the following 'unlock_page' */
316 WRITE_ONCE(pcl->compressed_pages[i], NULL);
317 detach_page_private(page);
323 int erofs_try_to_free_cached_page(struct page *page)
325 struct z_erofs_pcluster *const pcl = (void *)page_private(page);
326 int ret = 0; /* 0 - busy */
328 if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) {
331 DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
332 for (i = 0; i < pcl->pclusterpages; ++i) {
333 if (pcl->compressed_pages[i] == page) {
334 WRITE_ONCE(pcl->compressed_pages[i], NULL);
339 erofs_workgroup_unfreeze(&pcl->obj, 1);
342 detach_page_private(page);
347 /* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */
348 static bool z_erofs_try_inplace_io(struct z_erofs_decompress_frontend *fe,
351 struct z_erofs_pcluster *const pcl = fe->pcl;
353 while (fe->icpage_ptr > pcl->compressed_pages)
354 if (!cmpxchg(--fe->icpage_ptr, NULL, page))
359 /* callers must be with pcluster lock held */
360 static int z_erofs_attach_page(struct z_erofs_decompress_frontend *fe,
361 struct page *page, enum z_erofs_page_type type,
366 /* give priority for inplaceio */
367 if (fe->mode >= COLLECT_PRIMARY &&
368 type == Z_EROFS_PAGE_TYPE_EXCLUSIVE &&
369 z_erofs_try_inplace_io(fe, page))
372 ret = z_erofs_pagevec_enqueue(&fe->vector, page, type,
374 fe->pcl->vcnt += (unsigned int)ret;
375 return ret ? 0 : -EAGAIN;
378 static void z_erofs_try_to_claim_pcluster(struct z_erofs_decompress_frontend *f)
380 struct z_erofs_pcluster *pcl = f->pcl;
381 z_erofs_next_pcluster_t *owned_head = &f->owned_head;
383 /* type 1, nil pcluster (this pcluster doesn't belong to any chain.) */
384 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL,
385 *owned_head) == Z_EROFS_PCLUSTER_NIL) {
386 *owned_head = &pcl->next;
387 /* so we can attach this pcluster to our submission chain. */
388 f->mode = COLLECT_PRIMARY_FOLLOWED;
393 * type 2, link to the end of an existing open chain, be careful
394 * that its submission is controlled by the original attached chain.
396 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
397 *owned_head) == Z_EROFS_PCLUSTER_TAIL) {
398 *owned_head = Z_EROFS_PCLUSTER_TAIL;
399 f->mode = COLLECT_PRIMARY_HOOKED;
403 /* type 3, it belongs to a chain, but it isn't the end of the chain */
404 f->mode = COLLECT_PRIMARY;
407 static int z_erofs_lookup_pcluster(struct z_erofs_decompress_frontend *fe,
409 struct erofs_map_blocks *map)
411 struct z_erofs_pcluster *pcl = fe->pcl;
414 /* to avoid unexpected loop formed by corrupted images */
415 if (fe->owned_head == &pcl->next || pcl == fe->tailpcl) {
417 return -EFSCORRUPTED;
420 if (pcl->pageofs_out != (map->m_la & ~PAGE_MASK)) {
422 return -EFSCORRUPTED;
425 length = READ_ONCE(pcl->length);
426 if (length & Z_EROFS_PCLUSTER_FULL_LENGTH) {
427 if ((map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) > length) {
429 return -EFSCORRUPTED;
432 unsigned int llen = map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT;
434 if (map->m_flags & EROFS_MAP_FULL_MAPPED)
435 llen |= Z_EROFS_PCLUSTER_FULL_LENGTH;
437 while (llen > length &&
438 length != cmpxchg_relaxed(&pcl->length, length, llen)) {
440 length = READ_ONCE(pcl->length);
443 mutex_lock(&pcl->lock);
444 /* used to check tail merging loop due to corrupted images */
445 if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL)
448 z_erofs_try_to_claim_pcluster(fe);
452 static int z_erofs_register_pcluster(struct z_erofs_decompress_frontend *fe,
454 struct erofs_map_blocks *map)
456 bool ztailpacking = map->m_flags & EROFS_MAP_META;
457 struct z_erofs_pcluster *pcl;
458 struct erofs_workgroup *grp;
461 if (!(map->m_flags & EROFS_MAP_ENCODED)) {
463 return -EFSCORRUPTED;
466 /* no available pcluster, let's allocate one */
467 pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 :
468 map->m_plen >> PAGE_SHIFT);
472 atomic_set(&pcl->obj.refcount, 1);
473 pcl->algorithmformat = map->m_algorithmformat;
474 pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) |
475 (map->m_flags & EROFS_MAP_FULL_MAPPED ?
476 Z_EROFS_PCLUSTER_FULL_LENGTH : 0);
478 /* new pclusters should be claimed as type 1, primary and followed */
479 pcl->next = fe->owned_head;
480 pcl->pageofs_out = map->m_la & ~PAGE_MASK;
481 fe->mode = COLLECT_PRIMARY_FOLLOWED;
484 * lock all primary followed works before visible to others
485 * and mutex_trylock *never* fails for a new pcluster.
487 mutex_init(&pcl->lock);
488 DBG_BUGON(!mutex_trylock(&pcl->lock));
491 pcl->obj.index = 0; /* which indicates ztailpacking */
492 pcl->pageofs_in = erofs_blkoff(map->m_pa);
493 pcl->tailpacking_size = map->m_plen;
495 pcl->obj.index = map->m_pa >> PAGE_SHIFT;
497 grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj);
503 if (grp != &pcl->obj) {
504 fe->pcl = container_of(grp,
505 struct z_erofs_pcluster, obj);
510 /* used to check tail merging loop due to corrupted images */
511 if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL)
513 fe->owned_head = &pcl->next;
518 mutex_unlock(&pcl->lock);
519 z_erofs_free_pcluster(pcl);
523 static int z_erofs_collector_begin(struct z_erofs_decompress_frontend *fe,
525 struct erofs_map_blocks *map)
527 struct erofs_workgroup *grp;
532 /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */
533 DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_NIL);
534 DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
536 if (map->m_flags & EROFS_MAP_META) {
537 if ((map->m_pa & ~PAGE_MASK) + map->m_plen > PAGE_SIZE) {
539 return -EFSCORRUPTED;
544 grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT);
546 fe->pcl = container_of(grp, struct z_erofs_pcluster, obj);
549 ret = z_erofs_register_pcluster(fe, inode, map);
556 ret = z_erofs_lookup_pcluster(fe, inode, map);
558 erofs_workgroup_put(&fe->pcl->obj);
563 z_erofs_pagevec_ctor_init(&fe->vector, Z_EROFS_NR_INLINE_PAGEVECS,
564 fe->pcl->pagevec, fe->pcl->vcnt);
565 /* since file-backed online pages are traversed in reverse order */
566 fe->icpage_ptr = fe->pcl->compressed_pages +
567 z_erofs_pclusterpages(fe->pcl);
572 * keep in mind that no referenced pclusters will be freed
573 * only after a RCU grace period.
575 static void z_erofs_rcu_callback(struct rcu_head *head)
577 z_erofs_free_pcluster(container_of(head,
578 struct z_erofs_pcluster, rcu));
581 void erofs_workgroup_free_rcu(struct erofs_workgroup *grp)
583 struct z_erofs_pcluster *const pcl =
584 container_of(grp, struct z_erofs_pcluster, obj);
586 call_rcu(&pcl->rcu, z_erofs_rcu_callback);
589 static bool z_erofs_collector_end(struct z_erofs_decompress_frontend *fe)
591 struct z_erofs_pcluster *pcl = fe->pcl;
596 z_erofs_pagevec_ctor_exit(&fe->vector, false);
597 mutex_unlock(&pcl->lock);
600 * if all pending pages are added, don't hold its reference
601 * any longer if the pcluster isn't hosted by ourselves.
603 if (fe->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE)
604 erofs_workgroup_put(&pcl->obj);
610 static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe,
611 unsigned int cachestrategy,
614 if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED)
620 return cachestrategy >= EROFS_ZIP_CACHE_READAROUND &&
624 static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
625 struct page *page, struct page **pagepool)
627 struct inode *const inode = fe->inode;
628 struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
629 struct erofs_map_blocks *const map = &fe->map;
630 const loff_t offset = page_offset(page);
633 enum z_erofs_cache_alloctype cache_strategy;
634 enum z_erofs_page_type page_type;
635 unsigned int cur, end, spiltted, index;
638 /* register locked file pages as online pages in pack */
639 z_erofs_onlinepage_init(page);
646 if (offset + cur < map->m_la ||
647 offset + cur >= map->m_la + map->m_llen) {
648 erofs_dbg("out-of-range map @ pos %llu", offset + cur);
650 if (z_erofs_collector_end(fe))
651 fe->backmost = false;
652 map->m_la = offset + cur;
654 err = z_erofs_map_blocks_iter(inode, map, 0);
660 /* didn't get a valid pcluster previously (very rare) */
663 if (!(map->m_flags & EROFS_MAP_MAPPED))
666 err = z_erofs_collector_begin(fe, inode, map);
670 if (z_erofs_is_inline_pcluster(fe->pcl)) {
673 mp = erofs_read_metabuf(&fe->map.buf, inode->i_sb,
674 erofs_blknr(map->m_pa), EROFS_NO_KMAP);
677 erofs_err(inode->i_sb,
678 "failed to get inline page, err %d", err);
681 get_page(fe->map.buf.page);
682 WRITE_ONCE(fe->pcl->compressed_pages[0], fe->map.buf.page);
683 fe->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;
685 /* bind cache first when cached decompression is preferred */
686 if (should_alloc_managed_pages(fe, sbi->opt.cache_strategy,
688 cache_strategy = TRYALLOC;
690 cache_strategy = DONTALLOC;
692 z_erofs_bind_cache(fe, cache_strategy, pagepool);
696 * Ensure the current partial page belongs to this submit chain rather
697 * than other concurrent submit chains or the noio(bypass) chain since
698 * those chains are handled asynchronously thus the page cannot be used
699 * for inplace I/O or pagevec (should be processed in strict order.)
701 tight &= (fe->mode >= COLLECT_PRIMARY_HOOKED &&
702 fe->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE);
704 cur = end - min_t(unsigned int, offset + end - map->m_la, end);
705 if (!(map->m_flags & EROFS_MAP_MAPPED)) {
706 zero_user_segment(page, cur, end);
710 /* let's derive page type */
711 page_type = cur ? Z_EROFS_VLE_PAGE_TYPE_HEAD :
712 (!spiltted ? Z_EROFS_PAGE_TYPE_EXCLUSIVE :
713 (tight ? Z_EROFS_PAGE_TYPE_EXCLUSIVE :
714 Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED));
717 tight &= (fe->mode >= COLLECT_PRIMARY_FOLLOWED);
720 err = z_erofs_attach_page(fe, page, page_type,
721 fe->mode >= COLLECT_PRIMARY_FOLLOWED);
722 /* should allocate an additional short-lived page for pagevec */
723 if (err == -EAGAIN) {
724 struct page *const newpage =
725 alloc_page(GFP_NOFS | __GFP_NOFAIL);
727 set_page_private(newpage, Z_EROFS_SHORTLIVED_PAGE);
728 err = z_erofs_attach_page(fe, newpage,
729 Z_EROFS_PAGE_TYPE_EXCLUSIVE, true);
737 index = page->index - (map->m_la >> PAGE_SHIFT);
739 z_erofs_onlinepage_fixup(page, index, true);
741 /* bump up the number of spiltted parts of a page */
743 /* also update nr_pages */
744 fe->pcl->nr_pages = max_t(pgoff_t, fe->pcl->nr_pages, index + 1);
746 /* can be used for verification */
747 map->m_llen = offset + cur - map->m_la;
754 z_erofs_onlinepage_endio(page);
756 erofs_dbg("%s, finish page: %pK spiltted: %u map->m_llen %llu",
757 __func__, page, spiltted, map->m_llen);
760 /* if some error occurred while processing this page */
766 static bool z_erofs_get_sync_decompress_policy(struct erofs_sb_info *sbi,
767 unsigned int readahead_pages)
769 /* auto: enable for read_folio, disable for readahead */
770 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) &&
774 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_FORCE_ON) &&
775 (readahead_pages <= sbi->opt.max_sync_decompress_pages))
781 static bool z_erofs_page_is_invalidated(struct page *page)
783 return !page->mapping && !z_erofs_is_shortlived_page(page);
786 static int z_erofs_decompress_pcluster(struct super_block *sb,
787 struct z_erofs_pcluster *pcl,
788 struct page **pagepool)
790 struct erofs_sb_info *const sbi = EROFS_SB(sb);
791 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
792 struct z_erofs_pagevec_ctor ctor;
793 unsigned int i, inputsize, outputsize, llen, nr_pages;
794 struct page *pages_onstack[Z_EROFS_VMAP_ONSTACK_PAGES];
795 struct page **pages, **compressed_pages, *page;
797 enum z_erofs_page_type page_type;
798 bool overlapped, partial;
802 DBG_BUGON(!READ_ONCE(pcl->nr_pages));
804 mutex_lock(&pcl->lock);
805 nr_pages = pcl->nr_pages;
807 if (nr_pages <= Z_EROFS_VMAP_ONSTACK_PAGES) {
808 pages = pages_onstack;
809 } else if (nr_pages <= Z_EROFS_VMAP_GLOBAL_PAGES &&
810 mutex_trylock(&z_pagemap_global_lock)) {
811 pages = z_pagemap_global;
813 gfp_t gfp_flags = GFP_KERNEL;
815 if (nr_pages > Z_EROFS_VMAP_GLOBAL_PAGES)
816 gfp_flags |= __GFP_NOFAIL;
818 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
821 /* fallback to global pagemap for the lowmem scenario */
823 mutex_lock(&z_pagemap_global_lock);
824 pages = z_pagemap_global;
828 for (i = 0; i < nr_pages; ++i)
832 z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_NR_INLINE_PAGEVECS,
835 for (i = 0; i < pcl->vcnt; ++i) {
838 page = z_erofs_pagevec_dequeue(&ctor, &page_type);
840 /* all pages in pagevec ought to be valid */
842 DBG_BUGON(z_erofs_page_is_invalidated(page));
844 if (z_erofs_put_shortlivedpage(pagepool, page))
847 if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD)
850 pagenr = z_erofs_onlinepage_index(page);
852 DBG_BUGON(pagenr >= nr_pages);
855 * currently EROFS doesn't support multiref(dedup),
856 * so here erroring out one multiref page.
860 SetPageError(pages[pagenr]);
861 z_erofs_onlinepage_endio(pages[pagenr]);
864 pages[pagenr] = page;
866 z_erofs_pagevec_ctor_exit(&ctor, true);
869 compressed_pages = pcl->compressed_pages;
871 for (i = 0; i < pclusterpages; ++i) {
874 page = compressed_pages[i];
875 /* all compressed pages ought to be valid */
878 if (z_erofs_is_inline_pcluster(pcl)) {
879 if (!PageUptodate(page))
884 DBG_BUGON(z_erofs_page_is_invalidated(page));
885 if (!z_erofs_is_shortlived_page(page)) {
886 if (erofs_page_is_managed(sbi, page)) {
887 if (!PageUptodate(page))
893 * only if non-head page can be selected
894 * for inplace decompression
896 pagenr = z_erofs_onlinepage_index(page);
898 DBG_BUGON(pagenr >= nr_pages);
901 SetPageError(pages[pagenr]);
902 z_erofs_onlinepage_endio(pages[pagenr]);
905 pages[pagenr] = page;
910 /* PG_error needs checking for all non-managed pages */
911 if (PageError(page)) {
912 DBG_BUGON(PageUptodate(page));
920 llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT;
921 if (nr_pages << PAGE_SHIFT >= pcl->pageofs_out + llen) {
923 partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH);
925 outputsize = (nr_pages << PAGE_SHIFT) - pcl->pageofs_out;
929 if (z_erofs_is_inline_pcluster(pcl))
930 inputsize = pcl->tailpacking_size;
932 inputsize = pclusterpages * PAGE_SIZE;
934 err = z_erofs_decompress(&(struct z_erofs_decompress_req) {
936 .in = compressed_pages,
938 .pageofs_in = pcl->pageofs_in,
939 .pageofs_out = pcl->pageofs_out,
940 .inputsize = inputsize,
941 .outputsize = outputsize,
942 .alg = pcl->algorithmformat,
943 .inplace_io = overlapped,
944 .partial_decoding = partial
948 /* must handle all compressed pages before actual file pages */
949 if (z_erofs_is_inline_pcluster(pcl)) {
950 page = compressed_pages[0];
951 WRITE_ONCE(compressed_pages[0], NULL);
954 for (i = 0; i < pclusterpages; ++i) {
955 page = compressed_pages[i];
957 if (erofs_page_is_managed(sbi, page))
960 /* recycle all individual short-lived pages */
961 (void)z_erofs_put_shortlivedpage(pagepool, page);
962 WRITE_ONCE(compressed_pages[i], NULL);
966 for (i = 0; i < nr_pages; ++i) {
971 DBG_BUGON(z_erofs_page_is_invalidated(page));
973 /* recycle all individual short-lived pages */
974 if (z_erofs_put_shortlivedpage(pagepool, page))
980 z_erofs_onlinepage_endio(page);
983 if (pages == z_pagemap_global)
984 mutex_unlock(&z_pagemap_global_lock);
985 else if (pages != pages_onstack)
991 /* pcluster lock MUST be taken before the following line */
992 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL);
993 mutex_unlock(&pcl->lock);
997 static void z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io,
998 struct page **pagepool)
1000 z_erofs_next_pcluster_t owned = io->head;
1002 while (owned != Z_EROFS_PCLUSTER_TAIL_CLOSED) {
1003 struct z_erofs_pcluster *pcl;
1005 /* no possible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */
1006 DBG_BUGON(owned == Z_EROFS_PCLUSTER_TAIL);
1008 /* no possible that 'owned' equals NULL */
1009 DBG_BUGON(owned == Z_EROFS_PCLUSTER_NIL);
1011 pcl = container_of(owned, struct z_erofs_pcluster, next);
1012 owned = READ_ONCE(pcl->next);
1014 z_erofs_decompress_pcluster(io->sb, pcl, pagepool);
1015 erofs_workgroup_put(&pcl->obj);
1019 static void z_erofs_decompressqueue_work(struct work_struct *work)
1021 struct z_erofs_decompressqueue *bgq =
1022 container_of(work, struct z_erofs_decompressqueue, u.work);
1023 struct page *pagepool = NULL;
1025 DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
1026 z_erofs_decompress_queue(bgq, &pagepool);
1028 erofs_release_pages(&pagepool);
1032 static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io,
1033 bool sync, int bios)
1035 struct erofs_sb_info *const sbi = EROFS_SB(io->sb);
1037 /* wake up the caller thread for sync decompression */
1039 if (!atomic_add_return(bios, &io->pending_bios))
1040 complete(&io->u.done);
1045 if (atomic_add_return(bios, &io->pending_bios))
1047 /* Use workqueue and sync decompression for atomic contexts only */
1048 if (in_atomic() || irqs_disabled()) {
1049 queue_work(z_erofs_workqueue, &io->u.work);
1050 /* enable sync decompression for readahead */
1051 if (sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO)
1052 sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON;
1055 z_erofs_decompressqueue_work(&io->u.work);
1058 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl,
1060 struct page **pagepool,
1061 struct address_space *mc)
1063 const pgoff_t index = pcl->obj.index;
1064 gfp_t gfp = mapping_gfp_mask(mc);
1065 bool tocache = false;
1067 struct address_space *mapping;
1068 struct page *oldpage, *page;
1070 compressed_page_t t;
1074 page = READ_ONCE(pcl->compressed_pages[nr]);
1080 /* process the target tagged pointer */
1081 t = tagptr_init(compressed_page_t, page);
1082 justfound = tagptr_unfold_tags(t);
1083 page = tagptr_unfold_ptr(t);
1086 * preallocated cached pages, which is used to avoid direct reclaim
1087 * otherwise, it will go inplace I/O path instead.
1089 if (page->private == Z_EROFS_PREALLOCATED_PAGE) {
1090 WRITE_ONCE(pcl->compressed_pages[nr], page);
1091 set_page_private(page, 0);
1095 mapping = READ_ONCE(page->mapping);
1098 * file-backed online pages in plcuster are all locked steady,
1099 * therefore it is impossible for `mapping' to be NULL.
1101 if (mapping && mapping != mc)
1102 /* ought to be unmanaged pages */
1105 /* directly return for shortlived page as well */
1106 if (z_erofs_is_shortlived_page(page))
1111 /* only true if page reclaim goes wrong, should never happen */
1112 DBG_BUGON(justfound && PagePrivate(page));
1114 /* the page is still in manage cache */
1115 if (page->mapping == mc) {
1116 WRITE_ONCE(pcl->compressed_pages[nr], page);
1118 ClearPageError(page);
1119 if (!PagePrivate(page)) {
1121 * impossible to be !PagePrivate(page) for
1122 * the current restriction as well if
1123 * the page is already in compressed_pages[].
1125 DBG_BUGON(!justfound);
1128 set_page_private(page, (unsigned long)pcl);
1129 SetPagePrivate(page);
1132 /* no need to submit io if it is already up-to-date */
1133 if (PageUptodate(page)) {
1141 * the managed page has been truncated, it's unsafe to
1142 * reuse this one, let's allocate a new cache-managed page.
1144 DBG_BUGON(page->mapping);
1145 DBG_BUGON(!justfound);
1151 page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL);
1152 if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) {
1153 erofs_pagepool_add(pagepool, page);
1158 if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) {
1159 /* turn into temporary page if fails (1 ref) */
1160 set_page_private(page, Z_EROFS_SHORTLIVED_PAGE);
1163 attach_page_private(page, pcl);
1164 /* drop a refcount added by allocpage (then we have 2 refs here) */
1167 out: /* the only exit (for tracing and debugging) */
1171 static struct z_erofs_decompressqueue *
1172 jobqueue_init(struct super_block *sb,
1173 struct z_erofs_decompressqueue *fgq, bool *fg)
1175 struct z_erofs_decompressqueue *q;
1178 q = kvzalloc(sizeof(*q), GFP_KERNEL | __GFP_NOWARN);
1183 INIT_WORK(&q->u.work, z_erofs_decompressqueue_work);
1187 init_completion(&fgq->u.done);
1188 atomic_set(&fgq->pending_bios, 0);
1191 q->head = Z_EROFS_PCLUSTER_TAIL_CLOSED;
1195 /* define decompression jobqueue types */
1202 static void *jobqueueset_init(struct super_block *sb,
1203 struct z_erofs_decompressqueue *q[],
1204 struct z_erofs_decompressqueue *fgq, bool *fg)
1207 * if managed cache is enabled, bypass jobqueue is needed,
1208 * no need to read from device for all pclusters in this queue.
1210 q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL);
1211 q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, fg);
1213 return tagptr_cast_ptr(tagptr_fold(tagptr1_t, q[JQ_SUBMIT], *fg));
1216 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,
1217 z_erofs_next_pcluster_t qtail[],
1218 z_erofs_next_pcluster_t owned_head)
1220 z_erofs_next_pcluster_t *const submit_qtail = qtail[JQ_SUBMIT];
1221 z_erofs_next_pcluster_t *const bypass_qtail = qtail[JQ_BYPASS];
1223 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
1224 if (owned_head == Z_EROFS_PCLUSTER_TAIL)
1225 owned_head = Z_EROFS_PCLUSTER_TAIL_CLOSED;
1227 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED);
1229 WRITE_ONCE(*submit_qtail, owned_head);
1230 WRITE_ONCE(*bypass_qtail, &pcl->next);
1232 qtail[JQ_BYPASS] = &pcl->next;
1235 static void z_erofs_decompressqueue_endio(struct bio *bio)
1237 tagptr1_t t = tagptr_init(tagptr1_t, bio->bi_private);
1238 struct z_erofs_decompressqueue *q = tagptr_unfold_ptr(t);
1239 blk_status_t err = bio->bi_status;
1240 struct bio_vec *bvec;
1241 struct bvec_iter_all iter_all;
1243 bio_for_each_segment_all(bvec, bio, iter_all) {
1244 struct page *page = bvec->bv_page;
1246 DBG_BUGON(PageUptodate(page));
1247 DBG_BUGON(z_erofs_page_is_invalidated(page));
1252 if (erofs_page_is_managed(EROFS_SB(q->sb), page)) {
1254 SetPageUptodate(page);
1258 z_erofs_decompress_kickoff(q, tagptr_unfold_tags(t), -1);
1262 static void z_erofs_submit_queue(struct super_block *sb,
1263 struct z_erofs_decompress_frontend *f,
1264 struct page **pagepool,
1265 struct z_erofs_decompressqueue *fgq,
1268 struct erofs_sb_info *const sbi = EROFS_SB(sb);
1269 z_erofs_next_pcluster_t qtail[NR_JOBQUEUES];
1270 struct z_erofs_decompressqueue *q[NR_JOBQUEUES];
1272 z_erofs_next_pcluster_t owned_head = f->owned_head;
1273 /* bio is NULL initially, so no need to initialize last_{index,bdev} */
1275 struct block_device *last_bdev;
1276 unsigned int nr_bios = 0;
1277 struct bio *bio = NULL;
1279 bi_private = jobqueueset_init(sb, q, fgq, force_fg);
1280 qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head;
1281 qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head;
1283 /* by default, all need io submission */
1284 q[JQ_SUBMIT]->head = owned_head;
1287 struct erofs_map_dev mdev;
1288 struct z_erofs_pcluster *pcl;
1293 /* no possible 'owned_head' equals the following */
1294 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
1295 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_NIL);
1297 pcl = container_of(owned_head, struct z_erofs_pcluster, next);
1299 /* close the main owned chain at first */
1300 owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
1301 Z_EROFS_PCLUSTER_TAIL_CLOSED);
1302 if (z_erofs_is_inline_pcluster(pcl)) {
1303 move_to_bypass_jobqueue(pcl, qtail, owned_head);
1307 /* no device id here, thus it will always succeed */
1308 mdev = (struct erofs_map_dev) {
1309 .m_pa = blknr_to_addr(pcl->obj.index),
1311 (void)erofs_map_dev(sb, &mdev);
1313 cur = erofs_blknr(mdev.m_pa);
1314 end = cur + pcl->pclusterpages;
1319 page = pickup_page_for_submission(pcl, i++, pagepool,
1324 if (bio && (cur != last_index + 1 ||
1325 last_bdev != mdev.m_bdev)) {
1332 bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS,
1333 REQ_OP_READ, GFP_NOIO);
1334 bio->bi_end_io = z_erofs_decompressqueue_endio;
1336 last_bdev = mdev.m_bdev;
1337 bio->bi_iter.bi_sector = (sector_t)cur <<
1338 LOG_SECTORS_PER_BLOCK;
1339 bio->bi_private = bi_private;
1341 bio->bi_opf |= REQ_RAHEAD;
1345 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
1346 goto submit_bio_retry;
1350 } while (++cur < end);
1353 qtail[JQ_SUBMIT] = &pcl->next;
1355 move_to_bypass_jobqueue(pcl, qtail, owned_head);
1356 } while (owned_head != Z_EROFS_PCLUSTER_TAIL);
1362 * although background is preferred, no one is pending for submission.
1363 * don't issue workqueue for decompression but drop it directly instead.
1365 if (!*force_fg && !nr_bios) {
1366 kvfree(q[JQ_SUBMIT]);
1369 z_erofs_decompress_kickoff(q[JQ_SUBMIT], *force_fg, nr_bios);
1372 static void z_erofs_runqueue(struct super_block *sb,
1373 struct z_erofs_decompress_frontend *f,
1374 struct page **pagepool, bool force_fg)
1376 struct z_erofs_decompressqueue io[NR_JOBQUEUES];
1378 if (f->owned_head == Z_EROFS_PCLUSTER_TAIL)
1380 z_erofs_submit_queue(sb, f, pagepool, io, &force_fg);
1382 /* handle bypass queue (no i/o pclusters) immediately */
1383 z_erofs_decompress_queue(&io[JQ_BYPASS], pagepool);
1388 /* wait until all bios are completed */
1389 wait_for_completion_io(&io[JQ_SUBMIT].u.done);
1391 /* handle synchronous decompress queue in the caller context */
1392 z_erofs_decompress_queue(&io[JQ_SUBMIT], pagepool);
1396 * Since partial uptodate is still unimplemented for now, we have to use
1397 * approximate readmore strategies as a start.
1399 static void z_erofs_pcluster_readmore(struct z_erofs_decompress_frontend *f,
1400 struct readahead_control *rac,
1402 struct page **pagepool,
1405 struct inode *inode = f->inode;
1406 struct erofs_map_blocks *map = &f->map;
1412 err = z_erofs_map_blocks_iter(inode, map,
1413 EROFS_GET_BLOCKS_READMORE);
1417 /* expend ra for the trailing edge if readahead */
1419 loff_t newstart = readahead_pos(rac);
1421 cur = round_up(map->m_la + map->m_llen, PAGE_SIZE);
1422 readahead_expand(rac, newstart, cur - newstart);
1425 end = round_up(end, PAGE_SIZE);
1427 end = round_up(map->m_la, PAGE_SIZE);
1433 cur = map->m_la + map->m_llen - 1;
1434 while (cur >= end) {
1435 pgoff_t index = cur >> PAGE_SHIFT;
1438 page = erofs_grab_cache_page_nowait(inode->i_mapping, index);
1440 if (PageUptodate(page)) {
1443 err = z_erofs_do_read_page(f, page, pagepool);
1445 erofs_err(inode->i_sb,
1446 "readmore error at page %lu @ nid %llu",
1447 index, EROFS_I(inode)->nid);
1452 if (cur < PAGE_SIZE)
1454 cur = (index << PAGE_SHIFT) - 1;
1458 static int z_erofs_read_folio(struct file *file, struct folio *folio)
1460 struct page *page = &folio->page;
1461 struct inode *const inode = page->mapping->host;
1462 struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
1463 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
1464 struct page *pagepool = NULL;
1467 trace_erofs_readpage(page, false);
1468 f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT;
1470 z_erofs_pcluster_readmore(&f, NULL, f.headoffset + PAGE_SIZE - 1,
1472 err = z_erofs_do_read_page(&f, page, &pagepool);
1473 z_erofs_pcluster_readmore(&f, NULL, 0, &pagepool, false);
1475 (void)z_erofs_collector_end(&f);
1477 /* if some compressed cluster ready, need submit them anyway */
1478 z_erofs_runqueue(inode->i_sb, &f, &pagepool,
1479 z_erofs_get_sync_decompress_policy(sbi, 0));
1482 erofs_err(inode->i_sb, "failed to read, err [%d]", err);
1484 erofs_put_metabuf(&f.map.buf);
1485 erofs_release_pages(&pagepool);
1489 static void z_erofs_readahead(struct readahead_control *rac)
1491 struct inode *const inode = rac->mapping->host;
1492 struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
1493 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
1494 struct page *pagepool = NULL, *head = NULL, *page;
1495 unsigned int nr_pages;
1498 f.headoffset = readahead_pos(rac);
1500 z_erofs_pcluster_readmore(&f, rac, f.headoffset +
1501 readahead_length(rac) - 1, &pagepool, true);
1502 nr_pages = readahead_count(rac);
1503 trace_erofs_readpages(inode, readahead_index(rac), nr_pages, false);
1505 while ((page = readahead_page(rac))) {
1506 set_page_private(page, (unsigned long)head);
1511 struct page *page = head;
1514 /* traversal in reverse order */
1515 head = (void *)page_private(page);
1517 err = z_erofs_do_read_page(&f, page, &pagepool);
1519 erofs_err(inode->i_sb,
1520 "readahead error at page %lu @ nid %llu",
1521 page->index, EROFS_I(inode)->nid);
1524 z_erofs_pcluster_readmore(&f, rac, 0, &pagepool, false);
1525 (void)z_erofs_collector_end(&f);
1527 z_erofs_runqueue(inode->i_sb, &f, &pagepool,
1528 z_erofs_get_sync_decompress_policy(sbi, nr_pages));
1529 erofs_put_metabuf(&f.map.buf);
1530 erofs_release_pages(&pagepool);
1533 const struct address_space_operations z_erofs_aops = {
1534 .read_folio = z_erofs_read_folio,
1535 .readahead = z_erofs_readahead,