fs/erofs/fscache.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2022, Alibaba Cloud
   4  */
   5 #include <linux/fscache.h>
   6 #include "internal.h"
   7
   8 static struct netfs_io_request *erofs_fscache_alloc_request(struct address_space *mapping,
   9                                              loff_t start, size_t len)
  10 {
  11         struct netfs_io_request *rreq;
  12
  13         rreq = kzalloc(sizeof(struct netfs_io_request), GFP_KERNEL);
  14         if (!rreq)
  15                 return ERR_PTR(-ENOMEM);
  16
  17         rreq->start     = start;
  18         rreq->len       = len;
  19         rreq->mapping   = mapping;
  20         rreq->inode     = mapping->host;
  21         INIT_LIST_HEAD(&rreq->subrequests);
  22         refcount_set(&rreq->ref, 1);
  23         return rreq;
  24 }
  25
  26 static void erofs_fscache_put_request(struct netfs_io_request *rreq)
  27 {
  28         if (!refcount_dec_and_test(&rreq->ref))
  29                 return;
  30         if (rreq->cache_resources.ops)
  31                 rreq->cache_resources.ops->end_operation(&rreq->cache_resources);
  32         kfree(rreq);
  33 }
  34
  35 static void erofs_fscache_put_subrequest(struct netfs_io_subrequest *subreq)
  36 {
  37         if (!refcount_dec_and_test(&subreq->ref))
  38                 return;
  39         erofs_fscache_put_request(subreq->rreq);
  40         kfree(subreq);
  41 }
  42
  43 static void erofs_fscache_clear_subrequests(struct netfs_io_request *rreq)
  44 {
  45         struct netfs_io_subrequest *subreq;
  46
  47         while (!list_empty(&rreq->subrequests)) {
  48                 subreq = list_first_entry(&rreq->subrequests,
  49                                 struct netfs_io_subrequest, rreq_link);
  50                 list_del(&subreq->rreq_link);
  51                 erofs_fscache_put_subrequest(subreq);
  52         }
  53 }
  54
  55 static void erofs_fscache_rreq_unlock_folios(struct netfs_io_request *rreq)
  56 {
  57         struct netfs_io_subrequest *subreq;
  58         struct folio *folio;
  59         unsigned int iopos = 0;
  60         pgoff_t start_page = rreq->start / PAGE_SIZE;
  61         pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
  62         bool subreq_failed = false;
  63
  64         XA_STATE(xas, &rreq->mapping->i_pages, start_page);
  65
  66         subreq = list_first_entry(&rreq->subrequests,
  67                                   struct netfs_io_subrequest, rreq_link);
  68         subreq_failed = (subreq->error < 0);
  69
  70         rcu_read_lock();
  71         xas_for_each(&xas, folio, last_page) {
  72                 unsigned int pgpos =
  73                         (folio_index(folio) - start_page) * PAGE_SIZE;
  74                 unsigned int pgend = pgpos + folio_size(folio);
  75                 bool pg_failed = false;
  76
  77                 for (;;) {
  78                         if (!subreq) {
  79                                 pg_failed = true;
  80                                 break;
  81                         }
  82
  83                         pg_failed |= subreq_failed;
  84                         if (pgend < iopos + subreq->len)
  85                                 break;
  86
  87                         iopos += subreq->len;
  88                         if (!list_is_last(&subreq->rreq_link,
  89                                           &rreq->subrequests)) {
  90                                 subreq = list_next_entry(subreq, rreq_link);
  91                                 subreq_failed = (subreq->error < 0);
  92                         } else {
  93                                 subreq = NULL;
  94                                 subreq_failed = false;
  95                         }
  96                         if (pgend == iopos)
  97                                 break;
  98                 }
  99
 100                 if (!pg_failed)
 101                         folio_mark_uptodate(folio);
 102
 103                 folio_unlock(folio);
 104         }
 105         rcu_read_unlock();
 106 }
 107
 108 static void erofs_fscache_rreq_complete(struct netfs_io_request *rreq)
 109 {
 110         erofs_fscache_rreq_unlock_folios(rreq);
 111         erofs_fscache_clear_subrequests(rreq);
 112         erofs_fscache_put_request(rreq);
 113 }
 114
 115 static void erofc_fscache_subreq_complete(void *priv,
 116                 ssize_t transferred_or_error, bool was_async)
 117 {
 118         struct netfs_io_subrequest *subreq = priv;
 119         struct netfs_io_request *rreq = subreq->rreq;
 120
 121         if (IS_ERR_VALUE(transferred_or_error))
 122                 subreq->error = transferred_or_error;
 123
 124         if (atomic_dec_and_test(&rreq->nr_outstanding))
 125                 erofs_fscache_rreq_complete(rreq);
 126
 127         erofs_fscache_put_subrequest(subreq);
 128 }
 129
 130 /*
 131  * Read data from fscache and fill the read data into page cache described by
 132  * @rreq, which shall be both aligned with PAGE_SIZE. @pstart describes
 133  * the start physical address in the cache file.
 134  */
 135 static int erofs_fscache_read_folios_async(struct fscache_cookie *cookie,
 136                                 struct netfs_io_request *rreq, loff_t pstart)
 137 {
 138         enum netfs_io_source source;
 139         struct super_block *sb = rreq->mapping->host->i_sb;
 140         struct netfs_io_subrequest *subreq;
 141         struct netfs_cache_resources *cres = &rreq->cache_resources;
 142         struct iov_iter iter;
 143         loff_t start = rreq->start;
 144         size_t len = rreq->len;
 145         size_t done = 0;
 146         int ret;
 147
 148         atomic_set(&rreq->nr_outstanding, 1);
 149
 150         ret = fscache_begin_read_operation(cres, cookie);
 151         if (ret)
 152                 goto out;
 153
 154         while (done < len) {
 155                 subreq = kzalloc(sizeof(struct netfs_io_subrequest),
 156                                  GFP_KERNEL);
 157                 if (subreq) {
 158                         INIT_LIST_HEAD(&subreq->rreq_link);
 159                         refcount_set(&subreq->ref, 2);
 160                         subreq->rreq = rreq;
 161                         refcount_inc(&rreq->ref);
 162                 } else {
 163                         ret = -ENOMEM;
 164                         goto out;
 165                 }
 166
 167                 subreq->start = pstart + done;
 168                 subreq->len     =  len - done;
 169                 subreq->flags = 1 << NETFS_SREQ_ONDEMAND;
 170
 171                 list_add_tail(&subreq->rreq_link, &rreq->subrequests);
 172
 173                 source = cres->ops->prepare_read(subreq, LLONG_MAX);
 174                 if (WARN_ON(subreq->len == 0))
 175                         source = NETFS_INVALID_READ;
 176                 if (source != NETFS_READ_FROM_CACHE) {
 177                         erofs_err(sb, "failed to fscache prepare_read (source %d)",
 178                                   source);
 179                         ret = -EIO;
 180                         subreq->error = ret;
 181                         erofs_fscache_put_subrequest(subreq);
 182                         goto out;
 183                 }
 184
 185                 atomic_inc(&rreq->nr_outstanding);
 186
 187                 iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages,
 188                                 start + done, subreq->len);
 189
 190                 ret = fscache_read(cres, subreq->start, &iter,
 191                                    NETFS_READ_HOLE_FAIL,
 192                                    erofc_fscache_subreq_complete, subreq);
 193                 if (ret == -EIOCBQUEUED)
 194                         ret = 0;
 195                 if (ret) {
 196                         erofs_err(sb, "failed to fscache_read (ret %d)", ret);
 197                         goto out;
 198                 }
 199
 200                 done += subreq->len;
 201         }
 202 out:
 203         if (atomic_dec_and_test(&rreq->nr_outstanding))
 204                 erofs_fscache_rreq_complete(rreq);
 205
 206         return ret;
 207 }
 208
 209 static int erofs_fscache_meta_read_folio(struct file *data, struct folio *folio)
 210 {
 211         int ret;
 212         struct super_block *sb = folio_mapping(folio)->host->i_sb;
 213         struct netfs_io_request *rreq;
 214         struct erofs_map_dev mdev = {
 215                 .m_deviceid = 0,
 216                 .m_pa = folio_pos(folio),
 217         };
 218
 219         ret = erofs_map_dev(sb, &mdev);
 220         if (ret)
 221                 goto out;
 222
 223         rreq = erofs_fscache_alloc_request(folio_mapping(folio),
 224                                 folio_pos(folio), folio_size(folio));
 225         if (IS_ERR(rreq))
 226                 goto out;
 227
 228         return erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
 229                                 rreq, mdev.m_pa);
 230 out:
 231         folio_unlock(folio);
 232         return ret;
 233 }
 234
 235 static int erofs_fscache_read_folio_inline(struct folio *folio,
 236                                          struct erofs_map_blocks *map)
 237 {
 238         struct super_block *sb = folio_mapping(folio)->host->i_sb;
 239         struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
 240         erofs_blk_t blknr;
 241         size_t offset, len;
 242         void *src, *dst;
 243
 244         /* For tail packing layout, the offset may be non-zero. */
 245         offset = erofs_blkoff(map->m_pa);
 246         blknr = erofs_blknr(map->m_pa);
 247         len = map->m_llen;
 248
 249         src = erofs_read_metabuf(&buf, sb, blknr, EROFS_KMAP);
 250         if (IS_ERR(src))
 251                 return PTR_ERR(src);
 252
 253         dst = kmap_local_folio(folio, 0);
 254         memcpy(dst, src + offset, len);
 255         memset(dst + len, 0, PAGE_SIZE - len);
 256         kunmap_local(dst);
 257
 258         erofs_put_metabuf(&buf);
 259         return 0;
 260 }
 261
 262 static int erofs_fscache_read_folio(struct file *file, struct folio *folio)
 263 {
 264         struct inode *inode = folio_mapping(folio)->host;
 265         struct super_block *sb = inode->i_sb;
 266         struct erofs_map_blocks map;
 267         struct erofs_map_dev mdev;
 268         struct netfs_io_request *rreq;
 269         erofs_off_t pos;
 270         loff_t pstart;
 271         int ret;
 272
 273         DBG_BUGON(folio_size(folio) != EROFS_BLKSIZ);
 274
 275         pos = folio_pos(folio);
 276         map.m_la = pos;
 277
 278         ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
 279         if (ret)
 280                 goto out_unlock;
 281
 282         if (!(map.m_flags & EROFS_MAP_MAPPED)) {
 283                 folio_zero_range(folio, 0, folio_size(folio));
 284                 goto out_uptodate;
 285         }
 286
 287         if (map.m_flags & EROFS_MAP_META) {
 288                 ret = erofs_fscache_read_folio_inline(folio, &map);
 289                 goto out_uptodate;
 290         }
 291
 292         mdev = (struct erofs_map_dev) {
 293                 .m_deviceid = map.m_deviceid,
 294                 .m_pa = map.m_pa,
 295         };
 296
 297         ret = erofs_map_dev(sb, &mdev);
 298         if (ret)
 299                 goto out_unlock;
 300
 301
 302         rreq = erofs_fscache_alloc_request(folio_mapping(folio),
 303                                 folio_pos(folio), folio_size(folio));
 304         if (IS_ERR(rreq))
 305                 goto out_unlock;
 306
 307         pstart = mdev.m_pa + (pos - map.m_la);
 308         return erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
 309                                 rreq, pstart);
 310
 311 out_uptodate:
 312         if (!ret)
 313                 folio_mark_uptodate(folio);
 314 out_unlock:
 315         folio_unlock(folio);
 316         return ret;
 317 }
 318
 319 static void erofs_fscache_advance_folios(struct readahead_control *rac,
 320                                          size_t len, bool unlock)
 321 {
 322         while (len) {
 323                 struct folio *folio = readahead_folio(rac);
 324                 len -= folio_size(folio);
 325                 if (unlock) {
 326                         folio_mark_uptodate(folio);
 327                         folio_unlock(folio);
 328                 }
 329         }
 330 }
 331
 332 static void erofs_fscache_readahead(struct readahead_control *rac)
 333 {
 334         struct inode *inode = rac->mapping->host;
 335         struct super_block *sb = inode->i_sb;
 336         size_t len, count, done = 0;
 337         erofs_off_t pos;
 338         loff_t start, offset;
 339         int ret;
 340
 341         if (!readahead_count(rac))
 342                 return;
 343
 344         start = readahead_pos(rac);
 345         len = readahead_length(rac);
 346
 347         do {
 348                 struct erofs_map_blocks map;
 349                 struct erofs_map_dev mdev;
 350                 struct netfs_io_request *rreq;
 351
 352                 pos = start + done;
 353                 map.m_la = pos;
 354
 355                 ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
 356                 if (ret)
 357                         return;
 358
 359                 offset = start + done;
 360                 count = min_t(size_t, map.m_llen - (pos - map.m_la),
 361                               len - done);
 362
 363                 if (!(map.m_flags & EROFS_MAP_MAPPED)) {
 364                         struct iov_iter iter;
 365
 366                         iov_iter_xarray(&iter, READ, &rac->mapping->i_pages,
 367                                         offset, count);
 368                         iov_iter_zero(count, &iter);
 369
 370                         erofs_fscache_advance_folios(rac, count, true);
 371                         ret = count;
 372                         continue;
 373                 }
 374
 375                 if (map.m_flags & EROFS_MAP_META) {
 376                         struct folio *folio = readahead_folio(rac);
 377
 378                         ret = erofs_fscache_read_folio_inline(folio, &map);
 379                         if (!ret) {
 380                                 folio_mark_uptodate(folio);
 381                                 ret = folio_size(folio);
 382                         }
 383
 384                         folio_unlock(folio);
 385                         continue;
 386                 }
 387
 388                 mdev = (struct erofs_map_dev) {
 389                         .m_deviceid = map.m_deviceid,
 390                         .m_pa = map.m_pa,
 391                 };
 392                 ret = erofs_map_dev(sb, &mdev);
 393                 if (ret)
 394                         return;
 395
 396                 rreq = erofs_fscache_alloc_request(rac->mapping, offset, count);
 397                 if (IS_ERR(rreq))
 398                         return;
 399                 /*
 400                  * Drop the ref of folios here. Unlock them in
 401                  * rreq_unlock_folios() when rreq complete.
 402                  */
 403                 erofs_fscache_advance_folios(rac, count, false);
 404                 ret = erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
 405                                         rreq, mdev.m_pa + (pos - map.m_la));
 406                 if (!ret)
 407                         ret = count;
 408         } while (ret > 0 && ((done += ret) < len));
 409 }
 410
 411 static const struct address_space_operations erofs_fscache_meta_aops = {
 412         .read_folio = erofs_fscache_meta_read_folio,
 413 };
 414
 415 const struct address_space_operations erofs_fscache_access_aops = {
 416         .read_folio = erofs_fscache_read_folio,
 417         .readahead = erofs_fscache_readahead,
 418 };
 419
 420 int erofs_fscache_register_cookie(struct super_block *sb,
 421                                   struct erofs_fscache **fscache,
 422                                   char *name, bool need_inode)
 423 {
 424         struct fscache_volume *volume = EROFS_SB(sb)->volume;
 425         struct erofs_fscache *ctx;
 426         struct fscache_cookie *cookie;
 427         int ret;
 428
 429         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 430         if (!ctx)
 431                 return -ENOMEM;
 432
 433         cookie = fscache_acquire_cookie(volume, FSCACHE_ADV_WANT_CACHE_SIZE,
 434                                         name, strlen(name), NULL, 0, 0);
 435         if (!cookie) {
 436                 erofs_err(sb, "failed to get cookie for %s", name);
 437                 ret = -EINVAL;
 438                 goto err;
 439         }
 440
 441         fscache_use_cookie(cookie, false);
 442         ctx->cookie = cookie;
 443
 444         if (need_inode) {
 445                 struct inode *const inode = new_inode(sb);
 446
 447                 if (!inode) {
 448                         erofs_err(sb, "failed to get anon inode for %s", name);
 449                         ret = -ENOMEM;
 450                         goto err_cookie;
 451                 }
 452
 453                 set_nlink(inode, 1);
 454                 inode->i_size = OFFSET_MAX;
 455                 inode->i_mapping->a_ops = &erofs_fscache_meta_aops;
 456                 mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
 457
 458                 ctx->inode = inode;
 459         }
 460
 461         *fscache = ctx;
 462         return 0;
 463
 464 err_cookie:
 465         fscache_unuse_cookie(ctx->cookie, NULL, NULL);
 466         fscache_relinquish_cookie(ctx->cookie, false);
 467         ctx->cookie = NULL;
 468 err:
 469         kfree(ctx);
 470         return ret;
 471 }
 472
 473 void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache)
 474 {
 475         struct erofs_fscache *ctx = *fscache;
 476
 477         if (!ctx)
 478                 return;
 479
 480         fscache_unuse_cookie(ctx->cookie, NULL, NULL);
 481         fscache_relinquish_cookie(ctx->cookie, false);
 482         ctx->cookie = NULL;
 483
 484         iput(ctx->inode);
 485         ctx->inode = NULL;
 486
 487         kfree(ctx);
 488         *fscache = NULL;
 489 }
 490
 491 int erofs_fscache_register_fs(struct super_block *sb)
 492 {
 493         struct erofs_sb_info *sbi = EROFS_SB(sb);
 494         struct fscache_volume *volume;
 495         char *name;
 496         int ret = 0;
 497
 498         name = kasprintf(GFP_KERNEL, "erofs,%s", sbi->opt.fsid);
 499         if (!name)
 500                 return -ENOMEM;
 501
 502         volume = fscache_acquire_volume(name, NULL, NULL, 0);
 503         if (IS_ERR_OR_NULL(volume)) {
 504                 erofs_err(sb, "failed to register volume for %s", name);
 505                 ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP;
 506                 volume = NULL;
 507         }
 508
 509         sbi->volume = volume;
 510         kfree(name);
 511         return ret;
 512 }
 513
 514 void erofs_fscache_unregister_fs(struct super_block *sb)
 515 {
 516         struct erofs_sb_info *sbi = EROFS_SB(sb);
 517
 518         fscache_relinquish_volume(sbi->volume, NULL, false);
 519         sbi->volume = NULL;
 520 }