lib/iov_iter.c

   1 #include <linux/export.h>
   2 #include <linux/uio.h>
   3 #include <linux/pagemap.h>
   4 #include <linux/slab.h>
   5 #include <linux/vmalloc.h>
   6 #include <linux/splice.h>
   7 #include <net/checksum.h>
   8
   9 #define PIPE_PARANOIA /* for now */
  10
  11 #define iterate_iovec(i, n, __v, __p, skip, STEP) {     \
  12         size_t left;                                    \
  13         size_t wanted = n;                              \
  14         __p = i->iov;                                   \
  15         __v.iov_len = min(n, __p->iov_len - skip);      \
  16         if (likely(__v.iov_len)) {                      \
  17                 __v.iov_base = __p->iov_base + skip;    \
  18                 left = (STEP);                          \
  19                 __v.iov_len -= left;                    \
  20                 skip += __v.iov_len;                    \
  21                 n -= __v.iov_len;                       \
  22         } else {                                        \
  23                 left = 0;                               \
  24         }                                               \
  25         while (unlikely(!left && n)) {                  \
  26                 __p++;                                  \
  27                 __v.iov_len = min(n, __p->iov_len);     \
  28                 if (unlikely(!__v.iov_len))             \
  29                         continue;                       \
  30                 __v.iov_base = __p->iov_base;           \
  31                 left = (STEP);                          \
  32                 __v.iov_len -= left;                    \
  33                 skip = __v.iov_len;                     \
  34                 n -= __v.iov_len;                       \
  35         }                                               \
  36         n = wanted - n;                                 \
  37 }
  38
  39 #define iterate_kvec(i, n, __v, __p, skip, STEP) {      \
  40         size_t wanted = n;                              \
  41         __p = i->kvec;                                  \
  42         __v.iov_len = min(n, __p->iov_len - skip);      \
  43         if (likely(__v.iov_len)) {                      \
  44                 __v.iov_base = __p->iov_base + skip;    \
  45                 (void)(STEP);                           \
  46                 skip += __v.iov_len;                    \
  47                 n -= __v.iov_len;                       \
  48         }                                               \
  49         while (unlikely(n)) {                           \
  50                 __p++;                                  \
  51                 __v.iov_len = min(n, __p->iov_len);     \
  52                 if (unlikely(!__v.iov_len))             \
  53                         continue;                       \
  54                 __v.iov_base = __p->iov_base;           \
  55                 (void)(STEP);                           \
  56                 skip = __v.iov_len;                     \
  57                 n -= __v.iov_len;                       \
  58         }                                               \
  59         n = wanted;                                     \
  60 }
  61
  62 #define iterate_bvec(i, n, __v, __bi, skip, STEP) {     \
  63         struct bvec_iter __start;                       \
  64         __start.bi_size = n;                            \
  65         __start.bi_bvec_done = skip;                    \
  66         __start.bi_idx = 0;                             \
  67         for_each_bvec(__v, i->bvec, __bi, __start) {    \
  68                 if (!__v.bv_len)                        \
  69                         continue;                       \
  70                 (void)(STEP);                           \
  71         }                                               \
  72 }
  73
  74 #define iterate_all_kinds(i, n, v, I, B, K) {                   \
  75         size_t skip = i->iov_offset;                            \
  76         if (unlikely(i->type & ITER_BVEC)) {                    \
  77                 struct bio_vec v;                               \
  78                 struct bvec_iter __bi;                          \
  79                 iterate_bvec(i, n, v, __bi, skip, (B))          \
  80         } else if (unlikely(i->type & ITER_KVEC)) {             \
  81                 const struct kvec *kvec;                        \
  82                 struct kvec v;                                  \
  83                 iterate_kvec(i, n, v, kvec, skip, (K))          \
  84         } else {                                                \
  85                 const struct iovec *iov;                        \
  86                 struct iovec v;                                 \
  87                 iterate_iovec(i, n, v, iov, skip, (I))          \
  88         }                                                       \
  89 }
  90
  91 #define iterate_and_advance(i, n, v, I, B, K) {                 \
  92         if (unlikely(i->count < n))                             \
  93                 n = i->count;                                   \
  94         if (i->count) {                                         \
  95                 size_t skip = i->iov_offset;                    \
  96                 if (unlikely(i->type & ITER_BVEC)) {            \
  97                         const struct bio_vec *bvec = i->bvec;   \
  98                         struct bio_vec v;                       \
  99                         struct bvec_iter __bi;                  \
 100                         iterate_bvec(i, n, v, __bi, skip, (B))  \
 101                         i->bvec = __bvec_iter_bvec(i->bvec, __bi);      \
 102                         i->nr_segs -= i->bvec - bvec;           \
 103                         skip = __bi.bi_bvec_done;               \
 104                 } else if (unlikely(i->type & ITER_KVEC)) {     \
 105                         const struct kvec *kvec;                \
 106                         struct kvec v;                          \
 107                         iterate_kvec(i, n, v, kvec, skip, (K))  \
 108                         if (skip == kvec->iov_len) {            \
 109                                 kvec++;                         \
 110                                 skip = 0;                       \
 111                         }                                       \
 112                         i->nr_segs -= kvec - i->kvec;           \
 113                         i->kvec = kvec;                         \
 114                 } else {                                        \
 115                         const struct iovec *iov;                \
 116                         struct iovec v;                         \
 117                         iterate_iovec(i, n, v, iov, skip, (I))  \
 118                         if (skip == iov->iov_len) {             \
 119                                 iov++;                          \
 120                                 skip = 0;                       \
 121                         }                                       \
 122                         i->nr_segs -= iov - i->iov;             \
 123                         i->iov = iov;                           \
 124                 }                                               \
 125                 i->count -= n;                                  \
 126                 i->iov_offset = skip;                           \
 127         }                                                       \
 128 }
 129
 130 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
 131                          struct iov_iter *i)
 132 {
 133         size_t skip, copy, left, wanted;
 134         const struct iovec *iov;
 135         char __user *buf;
 136         void *kaddr, *from;
 137
 138         if (unlikely(bytes > i->count))
 139                 bytes = i->count;
 140
 141         if (unlikely(!bytes))
 142                 return 0;
 143
 144         wanted = bytes;
 145         iov = i->iov;
 146         skip = i->iov_offset;
 147         buf = iov->iov_base + skip;
 148         copy = min(bytes, iov->iov_len - skip);
 149
 150         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
 151                 kaddr = kmap_atomic(page);
 152                 from = kaddr + offset;
 153
 154                 /* first chunk, usually the only one */
 155                 left = __copy_to_user_inatomic(buf, from, copy);
 156                 copy -= left;
 157                 skip += copy;
 158                 from += copy;
 159                 bytes -= copy;
 160
 161                 while (unlikely(!left && bytes)) {
 162                         iov++;
 163                         buf = iov->iov_base;
 164                         copy = min(bytes, iov->iov_len);
 165                         left = __copy_to_user_inatomic(buf, from, copy);
 166                         copy -= left;
 167                         skip = copy;
 168                         from += copy;
 169                         bytes -= copy;
 170                 }
 171                 if (likely(!bytes)) {
 172                         kunmap_atomic(kaddr);
 173                         goto done;
 174                 }
 175                 offset = from - kaddr;
 176                 buf += copy;
 177                 kunmap_atomic(kaddr);
 178                 copy = min(bytes, iov->iov_len - skip);
 179         }
 180         /* Too bad - revert to non-atomic kmap */
 181
 182         kaddr = kmap(page);
 183         from = kaddr + offset;
 184         left = __copy_to_user(buf, from, copy);
 185         copy -= left;
 186         skip += copy;
 187         from += copy;
 188         bytes -= copy;
 189         while (unlikely(!left && bytes)) {
 190                 iov++;
 191                 buf = iov->iov_base;
 192                 copy = min(bytes, iov->iov_len);
 193                 left = __copy_to_user(buf, from, copy);
 194                 copy -= left;
 195                 skip = copy;
 196                 from += copy;
 197                 bytes -= copy;
 198         }
 199         kunmap(page);
 200
 201 done:
 202         if (skip == iov->iov_len) {
 203                 iov++;
 204                 skip = 0;
 205         }
 206         i->count -= wanted - bytes;
 207         i->nr_segs -= iov - i->iov;
 208         i->iov = iov;
 209         i->iov_offset = skip;
 210         return wanted - bytes;
 211 }
 212
 213 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
 214                          struct iov_iter *i)
 215 {
 216         size_t skip, copy, left, wanted;
 217         const struct iovec *iov;
 218         char __user *buf;
 219         void *kaddr, *to;
 220
 221         if (unlikely(bytes > i->count))
 222                 bytes = i->count;
 223
 224         if (unlikely(!bytes))
 225                 return 0;
 226
 227         wanted = bytes;
 228         iov = i->iov;
 229         skip = i->iov_offset;
 230         buf = iov->iov_base + skip;
 231         copy = min(bytes, iov->iov_len - skip);
 232
 233         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
 234                 kaddr = kmap_atomic(page);
 235                 to = kaddr + offset;
 236
 237                 /* first chunk, usually the only one */
 238                 left = __copy_from_user_inatomic(to, buf, copy);
 239                 copy -= left;
 240                 skip += copy;
 241                 to += copy;
 242                 bytes -= copy;
 243
 244                 while (unlikely(!left && bytes)) {
 245                         iov++;
 246                         buf = iov->iov_base;
 247                         copy = min(bytes, iov->iov_len);
 248                         left = __copy_from_user_inatomic(to, buf, copy);
 249                         copy -= left;
 250                         skip = copy;
 251                         to += copy;
 252                         bytes -= copy;
 253                 }
 254                 if (likely(!bytes)) {
 255                         kunmap_atomic(kaddr);
 256                         goto done;
 257                 }
 258                 offset = to - kaddr;
 259                 buf += copy;
 260                 kunmap_atomic(kaddr);
 261                 copy = min(bytes, iov->iov_len - skip);
 262         }
 263         /* Too bad - revert to non-atomic kmap */
 264
 265         kaddr = kmap(page);
 266         to = kaddr + offset;
 267         left = __copy_from_user(to, buf, copy);
 268         copy -= left;
 269         skip += copy;
 270         to += copy;
 271         bytes -= copy;
 272         while (unlikely(!left && bytes)) {
 273                 iov++;
 274                 buf = iov->iov_base;
 275                 copy = min(bytes, iov->iov_len);
 276                 left = __copy_from_user(to, buf, copy);
 277                 copy -= left;
 278                 skip = copy;
 279                 to += copy;
 280                 bytes -= copy;
 281         }
 282         kunmap(page);
 283
 284 done:
 285         if (skip == iov->iov_len) {
 286                 iov++;
 287                 skip = 0;
 288         }
 289         i->count -= wanted - bytes;
 290         i->nr_segs -= iov - i->iov;
 291         i->iov = iov;
 292         i->iov_offset = skip;
 293         return wanted - bytes;
 294 }
 295
 296 #ifdef PIPE_PARANOIA
 297 static bool sanity(const struct iov_iter *i)
 298 {
 299         struct pipe_inode_info *pipe = i->pipe;
 300         int idx = i->idx;
 301         int next = pipe->curbuf + pipe->nrbufs;
 302         if (i->iov_offset) {
 303                 struct pipe_buffer *p;
 304                 if (unlikely(!pipe->nrbufs))
 305                         goto Bad;       // pipe must be non-empty
 306                 if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
 307                         goto Bad;       // must be at the last buffer...
 308
 309                 p = &pipe->bufs[idx];
 310                 if (unlikely(p->offset + p->len != i->iov_offset))
 311                         goto Bad;       // ... at the end of segment
 312         } else {
 313                 if (idx != (next & (pipe->buffers - 1)))
 314                         goto Bad;       // must be right after the last buffer
 315         }
 316         return true;
 317 Bad:
 318         printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
 319         printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
 320                         pipe->curbuf, pipe->nrbufs, pipe->buffers);
 321         for (idx = 0; idx < pipe->buffers; idx++)
 322                 printk(KERN_ERR "[%p %p %d %d]\n",
 323                         pipe->bufs[idx].ops,
 324                         pipe->bufs[idx].page,
 325                         pipe->bufs[idx].offset,
 326                         pipe->bufs[idx].len);
 327         WARN_ON(1);
 328         return false;
 329 }
 330 #else
 331 #define sanity(i) true
 332 #endif
 333
 334 static inline int next_idx(int idx, struct pipe_inode_info *pipe)
 335 {
 336         return (idx + 1) & (pipe->buffers - 1);
 337 }
 338
 339 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
 340                          struct iov_iter *i)
 341 {
 342         struct pipe_inode_info *pipe = i->pipe;
 343         struct pipe_buffer *buf;
 344         size_t off;
 345         int idx;
 346
 347         if (unlikely(bytes > i->count))
 348                 bytes = i->count;
 349
 350         if (unlikely(!bytes))
 351                 return 0;
 352
 353         if (!sanity(i))
 354                 return 0;
 355
 356         off = i->iov_offset;
 357         idx = i->idx;
 358         buf = &pipe->bufs[idx];
 359         if (off) {
 360                 if (offset == off && buf->page == page) {
 361                         /* merge with the last one */
 362                         buf->len += bytes;
 363                         i->iov_offset += bytes;
 364                         goto out;
 365                 }
 366                 idx = next_idx(idx, pipe);
 367                 buf = &pipe->bufs[idx];
 368         }
 369         if (idx == pipe->curbuf && pipe->nrbufs)
 370                 return 0;
 371         pipe->nrbufs++;
 372         buf->ops = &page_cache_pipe_buf_ops;
 373         buf->flags = 0;
 374         get_page(buf->page = page);
 375         buf->offset = offset;
 376         buf->len = bytes;
 377         i->iov_offset = offset + bytes;
 378         i->idx = idx;
 379 out:
 380         i->count -= bytes;
 381         return bytes;
 382 }
 383
 384 /*
 385  * Fault in one or more iovecs of the given iov_iter, to a maximum length of
 386  * bytes.  For each iovec, fault in each page that constitutes the iovec.
 387  *
 388  * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
 389  * because it is an invalid address).
 390  */
 391 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
 392 {
 393         size_t skip = i->iov_offset;
 394         const struct iovec *iov;
 395         int err;
 396         struct iovec v;
 397
 398         if (iter_is_iovec(i)) {
 399                 iterate_iovec(i, bytes, v, iov, skip, ({
 400                         err = fault_in_pages_readable(v.iov_base, v.iov_len);
 401                         if (unlikely(err))
 402                         return err;
 403                 0;}))
 404         }
 405         return 0;
 406 }
 407 EXPORT_SYMBOL(iov_iter_fault_in_readable);
 408
 409 void iov_iter_init(struct iov_iter *i, int direction,
 410                         const struct iovec *iov, unsigned long nr_segs,
 411                         size_t count)
 412 {
 413         /* It will get better.  Eventually... */
 414         if (segment_eq(get_fs(), KERNEL_DS)) {
 415                 direction |= ITER_KVEC;
 416                 i->type = direction;
 417                 i->kvec = (struct kvec *)iov;
 418         } else {
 419                 i->type = direction;
 420                 i->iov = iov;
 421         }
 422         i->nr_segs = nr_segs;
 423         i->iov_offset = 0;
 424         i->count = count;
 425 }
 426 EXPORT_SYMBOL(iov_iter_init);
 427
 428 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
 429 {
 430         char *from = kmap_atomic(page);
 431         memcpy(to, from + offset, len);
 432         kunmap_atomic(from);
 433 }
 434
 435 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
 436 {
 437         char *to = kmap_atomic(page);
 438         memcpy(to + offset, from, len);
 439         kunmap_atomic(to);
 440 }
 441
 442 static void memzero_page(struct page *page, size_t offset, size_t len)
 443 {
 444         char *addr = kmap_atomic(page);
 445         memset(addr + offset, 0, len);
 446         kunmap_atomic(addr);
 447 }
 448
 449 static inline bool allocated(struct pipe_buffer *buf)
 450 {
 451         return buf->ops == &default_pipe_buf_ops;
 452 }
 453
 454 static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
 455 {
 456         size_t off = i->iov_offset;
 457         int idx = i->idx;
 458         if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
 459                 idx = next_idx(idx, i->pipe);
 460                 off = 0;
 461         }
 462         *idxp = idx;
 463         *offp = off;
 464 }
 465
 466 static size_t push_pipe(struct iov_iter *i, size_t size,
 467                         int *idxp, size_t *offp)
 468 {
 469         struct pipe_inode_info *pipe = i->pipe;
 470         size_t off;
 471         int idx;
 472         ssize_t left;
 473
 474         if (unlikely(size > i->count))
 475                 size = i->count;
 476         if (unlikely(!size))
 477                 return 0;
 478
 479         left = size;
 480         data_start(i, &idx, &off);
 481         *idxp = idx;
 482         *offp = off;
 483         if (off) {
 484                 left -= PAGE_SIZE - off;
 485                 if (left <= 0) {
 486                         pipe->bufs[idx].len += size;
 487                         return size;
 488                 }
 489                 pipe->bufs[idx].len = PAGE_SIZE;
 490                 idx = next_idx(idx, pipe);
 491         }
 492         while (idx != pipe->curbuf || !pipe->nrbufs) {
 493                 struct page *page = alloc_page(GFP_USER);
 494                 if (!page)
 495                         break;
 496                 pipe->nrbufs++;
 497                 pipe->bufs[idx].ops = &default_pipe_buf_ops;
 498                 pipe->bufs[idx].flags = 0;
 499                 pipe->bufs[idx].page = page;
 500                 pipe->bufs[idx].offset = 0;
 501                 if (left <= PAGE_SIZE) {
 502                         pipe->bufs[idx].len = left;
 503                         return size;
 504                 }
 505                 pipe->bufs[idx].len = PAGE_SIZE;
 506                 left -= PAGE_SIZE;
 507                 idx = next_idx(idx, pipe);
 508         }
 509         return size - left;
 510 }
 511
 512 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
 513                                 struct iov_iter *i)
 514 {
 515         struct pipe_inode_info *pipe = i->pipe;
 516         size_t n, off;
 517         int idx;
 518
 519         if (!sanity(i))
 520                 return 0;
 521
 522         bytes = n = push_pipe(i, bytes, &idx, &off);
 523         if (unlikely(!n))
 524                 return 0;
 525         for ( ; n; idx = next_idx(idx, pipe), off = 0) {
 526                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
 527                 memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
 528                 i->idx = idx;
 529                 i->iov_offset = off + chunk;
 530                 n -= chunk;
 531                 addr += chunk;
 532         }
 533         i->count -= bytes;
 534         return bytes;
 535 }
 536
 537 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 538 {
 539         const char *from = addr;
 540         if (unlikely(i->type & ITER_PIPE))
 541                 return copy_pipe_to_iter(addr, bytes, i);
 542         iterate_and_advance(i, bytes, v,
 543                 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
 544                                v.iov_len),
 545                 memcpy_to_page(v.bv_page, v.bv_offset,
 546                                (from += v.bv_len) - v.bv_len, v.bv_len),
 547                 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
 548         )
 549
 550         return bytes;
 551 }
 552 EXPORT_SYMBOL(copy_to_iter);
 553
 554 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
 555 {
 556         char *to = addr;
 557         if (unlikely(i->type & ITER_PIPE)) {
 558                 WARN_ON(1);
 559                 return 0;
 560         }
 561         iterate_and_advance(i, bytes, v,
 562                 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
 563                                  v.iov_len),
 564                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 565                                  v.bv_offset, v.bv_len),
 566                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 567         )
 568
 569         return bytes;
 570 }
 571 EXPORT_SYMBOL(copy_from_iter);
 572
 573 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
 574 {
 575         char *to = addr;
 576         if (unlikely(i->type & ITER_PIPE)) {
 577                 WARN_ON(1);
 578                 return 0;
 579         }
 580         iterate_and_advance(i, bytes, v,
 581                 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
 582                                          v.iov_base, v.iov_len),
 583                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 584                                  v.bv_offset, v.bv_len),
 585                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 586         )
 587
 588         return bytes;
 589 }
 590 EXPORT_SYMBOL(copy_from_iter_nocache);
 591
 592 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
 593                          struct iov_iter *i)
 594 {
 595         if (i->type & (ITER_BVEC|ITER_KVEC)) {
 596                 void *kaddr = kmap_atomic(page);
 597                 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
 598                 kunmap_atomic(kaddr);
 599                 return wanted;
 600         } else if (likely(!(i->type & ITER_PIPE)))
 601                 return copy_page_to_iter_iovec(page, offset, bytes, i);
 602         else
 603                 return copy_page_to_iter_pipe(page, offset, bytes, i);
 604 }
 605 EXPORT_SYMBOL(copy_page_to_iter);
 606
 607 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
 608                          struct iov_iter *i)
 609 {
 610         if (unlikely(i->type & ITER_PIPE)) {
 611                 WARN_ON(1);
 612                 return 0;
 613         }
 614         if (i->type & (ITER_BVEC|ITER_KVEC)) {
 615                 void *kaddr = kmap_atomic(page);
 616                 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
 617                 kunmap_atomic(kaddr);
 618                 return wanted;
 619         } else
 620                 return copy_page_from_iter_iovec(page, offset, bytes, i);
 621 }
 622 EXPORT_SYMBOL(copy_page_from_iter);
 623
 624 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
 625 {
 626         struct pipe_inode_info *pipe = i->pipe;
 627         size_t n, off;
 628         int idx;
 629
 630         if (!sanity(i))
 631                 return 0;
 632
 633         bytes = n = push_pipe(i, bytes, &idx, &off);
 634         if (unlikely(!n))
 635                 return 0;
 636
 637         for ( ; n; idx = next_idx(idx, pipe), off = 0) {
 638                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
 639                 memzero_page(pipe->bufs[idx].page, off, chunk);
 640                 i->idx = idx;
 641                 i->iov_offset = off + chunk;
 642                 n -= chunk;
 643         }
 644         i->count -= bytes;
 645         return bytes;
 646 }
 647
 648 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
 649 {
 650         if (unlikely(i->type & ITER_PIPE))
 651                 return pipe_zero(bytes, i);
 652         iterate_and_advance(i, bytes, v,
 653                 __clear_user(v.iov_base, v.iov_len),
 654                 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
 655                 memset(v.iov_base, 0, v.iov_len)
 656         )
 657
 658         return bytes;
 659 }
 660 EXPORT_SYMBOL(iov_iter_zero);
 661
 662 size_t iov_iter_copy_from_user_atomic(struct page *page,
 663                 struct iov_iter *i, unsigned long offset, size_t bytes)
 664 {
 665         char *kaddr = kmap_atomic(page), *p = kaddr + offset;
 666         if (unlikely(i->type & ITER_PIPE)) {
 667                 kunmap_atomic(kaddr);
 668                 WARN_ON(1);
 669                 return 0;
 670         }
 671         iterate_all_kinds(i, bytes, v,
 672                 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
 673                                           v.iov_base, v.iov_len),
 674                 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
 675                                  v.bv_offset, v.bv_len),
 676                 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 677         )
 678         kunmap_atomic(kaddr);
 679         return bytes;
 680 }
 681 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
 682
 683 static inline void pipe_truncate(struct iov_iter *i)
 684 {
 685         struct pipe_inode_info *pipe = i->pipe;
 686         if (pipe->nrbufs) {
 687                 size_t off = i->iov_offset;
 688                 int idx = i->idx;
 689                 int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
 690                 if (off) {
 691                         pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
 692                         idx = next_idx(idx, pipe);
 693                         nrbufs++;
 694                 }
 695                 while (pipe->nrbufs > nrbufs) {
 696                         pipe_buf_release(pipe, &pipe->bufs[idx]);
 697                         idx = next_idx(idx, pipe);
 698                         pipe->nrbufs--;
 699                 }
 700         }
 701 }
 702
 703 static void pipe_advance(struct iov_iter *i, size_t size)
 704 {
 705         struct pipe_inode_info *pipe = i->pipe;
 706         if (unlikely(i->count < size))
 707                 size = i->count;
 708         if (size) {
 709                 struct pipe_buffer *buf;
 710                 size_t off = i->iov_offset, left = size;
 711                 int idx = i->idx;
 712                 if (off) /* make it relative to the beginning of buffer */
 713                         left += off - pipe->bufs[idx].offset;
 714                 while (1) {
 715                         buf = &pipe->bufs[idx];
 716                         if (left <= buf->len)
 717                                 break;
 718                         left -= buf->len;
 719                         idx = next_idx(idx, pipe);
 720                 }
 721                 i->idx = idx;
 722                 i->iov_offset = buf->offset + left;
 723         }
 724         i->count -= size;
 725         /* ... and discard everything past that point */
 726         pipe_truncate(i);
 727 }
 728
 729 void iov_iter_advance(struct iov_iter *i, size_t size)
 730 {
 731         if (unlikely(i->type & ITER_PIPE)) {
 732                 pipe_advance(i, size);
 733                 return;
 734         }
 735         iterate_and_advance(i, size, v, 0, 0, 0)
 736 }
 737 EXPORT_SYMBOL(iov_iter_advance);
 738
 739 void iov_iter_revert(struct iov_iter *i, size_t unroll)
 740 {
 741         if (!unroll)
 742                 return;
 743         i->count += unroll;
 744         if (unlikely(i->type & ITER_PIPE)) {
 745                 struct pipe_inode_info *pipe = i->pipe;
 746                 int idx = i->idx;
 747                 size_t off = i->iov_offset;
 748                 while (1) {
 749                         size_t n = off - pipe->bufs[idx].offset;
 750                         if (unroll < n) {
 751                                 off -= (n - unroll);
 752                                 break;
 753                         }
 754                         unroll -= n;
 755                         if (!unroll && idx == i->start_idx) {
 756                                 off = 0;
 757                                 break;
 758                         }
 759                         if (!idx--)
 760                                 idx = pipe->buffers - 1;
 761                         off = pipe->bufs[idx].offset + pipe->bufs[idx].len;
 762                 }
 763                 i->iov_offset = off;
 764                 i->idx = idx;
 765                 pipe_truncate(i);
 766                 return;
 767         }
 768         if (unroll <= i->iov_offset) {
 769                 i->iov_offset -= unroll;
 770                 return;
 771         }
 772         unroll -= i->iov_offset;
 773         if (i->type & ITER_BVEC) {
 774                 const struct bio_vec *bvec = i->bvec;
 775                 while (1) {
 776                         size_t n = (--bvec)->bv_len;
 777                         i->nr_segs++;
 778                         if (unroll <= n) {
 779                                 i->bvec = bvec;
 780                                 i->iov_offset = n - unroll;
 781                                 return;
 782                         }
 783                         unroll -= n;
 784                 }
 785         } else { /* same logics for iovec and kvec */
 786                 const struct iovec *iov = i->iov;
 787                 while (1) {
 788                         size_t n = (--iov)->iov_len;
 789                         i->nr_segs++;
 790                         if (unroll <= n) {
 791                                 i->iov = iov;
 792                                 i->iov_offset = n - unroll;
 793                                 return;
 794                         }
 795                         unroll -= n;
 796                 }
 797         }
 798 }
 799 EXPORT_SYMBOL(iov_iter_revert);
 800
 801 /*
 802  * Return the count of just the current iov_iter segment.
 803  */
 804 size_t iov_iter_single_seg_count(const struct iov_iter *i)
 805 {
 806         if (unlikely(i->type & ITER_PIPE))
 807                 return i->count;        // it is a silly place, anyway
 808         if (i->nr_segs == 1)
 809                 return i->count;
 810         else if (i->type & ITER_BVEC)
 811                 return min(i->count, i->bvec->bv_len - i->iov_offset);
 812         else
 813                 return min(i->count, i->iov->iov_len - i->iov_offset);
 814 }
 815 EXPORT_SYMBOL(iov_iter_single_seg_count);
 816
 817 void iov_iter_kvec(struct iov_iter *i, int direction,
 818                         const struct kvec *kvec, unsigned long nr_segs,
 819                         size_t count)
 820 {
 821         BUG_ON(!(direction & ITER_KVEC));
 822         i->type = direction;
 823         i->kvec = kvec;
 824         i->nr_segs = nr_segs;
 825         i->iov_offset = 0;
 826         i->count = count;
 827 }
 828 EXPORT_SYMBOL(iov_iter_kvec);
 829
 830 void iov_iter_bvec(struct iov_iter *i, int direction,
 831                         const struct bio_vec *bvec, unsigned long nr_segs,
 832                         size_t count)
 833 {
 834         BUG_ON(!(direction & ITER_BVEC));
 835         i->type = direction;
 836         i->bvec = bvec;
 837         i->nr_segs = nr_segs;
 838         i->iov_offset = 0;
 839         i->count = count;
 840 }
 841 EXPORT_SYMBOL(iov_iter_bvec);
 842
 843 void iov_iter_pipe(struct iov_iter *i, int direction,
 844                         struct pipe_inode_info *pipe,
 845                         size_t count)
 846 {
 847         BUG_ON(direction != ITER_PIPE);
 848         WARN_ON(pipe->nrbufs == pipe->buffers);
 849         i->type = direction;
 850         i->pipe = pipe;
 851         i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
 852         i->iov_offset = 0;
 853         i->count = count;
 854         i->start_idx = i->idx;
 855 }
 856 EXPORT_SYMBOL(iov_iter_pipe);
 857
 858 unsigned long iov_iter_alignment(const struct iov_iter *i)
 859 {
 860         unsigned long res = 0;
 861         size_t size = i->count;
 862
 863         if (!size)
 864                 return 0;
 865
 866         if (unlikely(i->type & ITER_PIPE)) {
 867                 if (i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
 868                         return size | i->iov_offset;
 869                 return size;
 870         }
 871         iterate_all_kinds(i, size, v,
 872                 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
 873                 res |= v.bv_offset | v.bv_len,
 874                 res |= (unsigned long)v.iov_base | v.iov_len
 875         )
 876         return res;
 877 }
 878 EXPORT_SYMBOL(iov_iter_alignment);
 879
 880 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
 881 {
 882         unsigned long res = 0;
 883         size_t size = i->count;
 884         if (!size)
 885                 return 0;
 886
 887         if (unlikely(i->type & ITER_PIPE)) {
 888                 WARN_ON(1);
 889                 return ~0U;
 890         }
 891
 892         iterate_all_kinds(i, size, v,
 893                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
 894                         (size != v.iov_len ? size : 0), 0),
 895                 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
 896                         (size != v.bv_len ? size : 0)),
 897                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
 898                         (size != v.iov_len ? size : 0))
 899                 );
 900                 return res;
 901 }
 902 EXPORT_SYMBOL(iov_iter_gap_alignment);
 903
 904 static inline size_t __pipe_get_pages(struct iov_iter *i,
 905                                 size_t maxsize,
 906                                 struct page **pages,
 907                                 int idx,
 908                                 size_t *start)
 909 {
 910         struct pipe_inode_info *pipe = i->pipe;
 911         ssize_t n = push_pipe(i, maxsize, &idx, start);
 912         if (!n)
 913                 return -EFAULT;
 914
 915         maxsize = n;
 916         n += *start;
 917         while (n > 0) {
 918                 get_page(*pages++ = pipe->bufs[idx].page);
 919                 idx = next_idx(idx, pipe);
 920                 n -= PAGE_SIZE;
 921         }
 922
 923         return maxsize;
 924 }
 925
 926 static ssize_t pipe_get_pages(struct iov_iter *i,
 927                    struct page **pages, size_t maxsize, unsigned maxpages,
 928                    size_t *start)
 929 {
 930         unsigned npages;
 931         size_t capacity;
 932         int idx;
 933
 934         if (!sanity(i))
 935                 return -EFAULT;
 936
 937         data_start(i, &idx, start);
 938         /* some of this one + all after this one */
 939         npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
 940         capacity = min(npages,maxpages) * PAGE_SIZE - *start;
 941
 942         return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
 943 }
 944
 945 ssize_t iov_iter_get_pages(struct iov_iter *i,
 946                    struct page **pages, size_t maxsize, unsigned maxpages,
 947                    size_t *start)
 948 {
 949         if (maxsize > i->count)
 950                 maxsize = i->count;
 951
 952         if (!maxsize)
 953                 return 0;
 954
 955         if (unlikely(i->type & ITER_PIPE))
 956                 return pipe_get_pages(i, pages, maxsize, maxpages, start);
 957         iterate_all_kinds(i, maxsize, v, ({
 958                 unsigned long addr = (unsigned long)v.iov_base;
 959                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
 960                 int n;
 961                 int res;
 962
 963                 if (len > maxpages * PAGE_SIZE)
 964                         len = maxpages * PAGE_SIZE;
 965                 addr &= ~(PAGE_SIZE - 1);
 966                 n = DIV_ROUND_UP(len, PAGE_SIZE);
 967                 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
 968                 if (unlikely(res < 0))
 969                         return res;
 970                 return (res == n ? len : res * PAGE_SIZE) - *start;
 971         0;}),({
 972                 /* can't be more than PAGE_SIZE */
 973                 *start = v.bv_offset;
 974                 get_page(*pages = v.bv_page);
 975                 return v.bv_len;
 976         }),({
 977                 return -EFAULT;
 978         })
 979         )
 980         return 0;
 981 }
 982 EXPORT_SYMBOL(iov_iter_get_pages);
 983
 984 static struct page **get_pages_array(size_t n)
 985 {
 986         struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
 987         if (!p)
 988                 p = vmalloc(n * sizeof(struct page *));
 989         return p;
 990 }
 991
 992 static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
 993                    struct page ***pages, size_t maxsize,
 994                    size_t *start)
 995 {
 996         struct page **p;
 997         size_t n;
 998         int idx;
 999         int npages;
1000
1001         if (!sanity(i))
1002                 return -EFAULT;
1003
1004         data_start(i, &idx, start);
1005         /* some of this one + all after this one */
1006         npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
1007         n = npages * PAGE_SIZE - *start;
1008         if (maxsize > n)
1009                 maxsize = n;
1010         else
1011                 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
1012         p = get_pages_array(npages);
1013         if (!p)
1014                 return -ENOMEM;
1015         n = __pipe_get_pages(i, maxsize, p, idx, start);
1016         if (n > 0)
1017                 *pages = p;
1018         else
1019                 kvfree(p);
1020         return n;
1021 }
1022
1023 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
1024                    struct page ***pages, size_t maxsize,
1025                    size_t *start)
1026 {
1027         struct page **p;
1028
1029         if (maxsize > i->count)
1030                 maxsize = i->count;
1031
1032         if (!maxsize)
1033                 return 0;
1034
1035         if (unlikely(i->type & ITER_PIPE))
1036                 return pipe_get_pages_alloc(i, pages, maxsize, start);
1037         iterate_all_kinds(i, maxsize, v, ({
1038                 unsigned long addr = (unsigned long)v.iov_base;
1039                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1040                 int n;
1041                 int res;
1042
1043                 addr &= ~(PAGE_SIZE - 1);
1044                 n = DIV_ROUND_UP(len, PAGE_SIZE);
1045                 p = get_pages_array(n);
1046                 if (!p)
1047                         return -ENOMEM;
1048                 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
1049                 if (unlikely(res < 0)) {
1050                         kvfree(p);
1051                         return res;
1052                 }
1053                 *pages = p;
1054                 return (res == n ? len : res * PAGE_SIZE) - *start;
1055         0;}),({
1056                 /* can't be more than PAGE_SIZE */
1057                 *start = v.bv_offset;
1058                 *pages = p = get_pages_array(1);
1059                 if (!p)
1060                         return -ENOMEM;
1061                 get_page(*p = v.bv_page);
1062                 return v.bv_len;
1063         }),({
1064                 return -EFAULT;
1065         })
1066         )
1067         return 0;
1068 }
1069 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
1070
1071 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1072                                struct iov_iter *i)
1073 {
1074         char *to = addr;
1075         __wsum sum, next;
1076         size_t off = 0;
1077         sum = *csum;
1078         if (unlikely(i->type & ITER_PIPE)) {
1079                 WARN_ON(1);
1080                 return 0;
1081         }
1082         iterate_and_advance(i, bytes, v, ({
1083                 int err = 0;
1084                 next = csum_and_copy_from_user(v.iov_base,
1085                                                (to += v.iov_len) - v.iov_len,
1086                                                v.iov_len, 0, &err);
1087                 if (!err) {
1088                         sum = csum_block_add(sum, next, off);
1089                         off += v.iov_len;
1090                 }
1091                 err ? v.iov_len : 0;
1092         }), ({
1093                 char *p = kmap_atomic(v.bv_page);
1094                 next = csum_partial_copy_nocheck(p + v.bv_offset,
1095                                                  (to += v.bv_len) - v.bv_len,
1096                                                  v.bv_len, 0);
1097                 kunmap_atomic(p);
1098                 sum = csum_block_add(sum, next, off);
1099                 off += v.bv_len;
1100         }),({
1101                 next = csum_partial_copy_nocheck(v.iov_base,
1102                                                  (to += v.iov_len) - v.iov_len,
1103                                                  v.iov_len, 0);
1104                 sum = csum_block_add(sum, next, off);
1105                 off += v.iov_len;
1106         })
1107         )
1108         *csum = sum;
1109         return bytes;
1110 }
1111 EXPORT_SYMBOL(csum_and_copy_from_iter);
1112
1113 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
1114                              struct iov_iter *i)
1115 {
1116         const char *from = addr;
1117         __wsum sum, next;
1118         size_t off = 0;
1119         sum = *csum;
1120         if (unlikely(i->type & ITER_PIPE)) {
1121                 WARN_ON(1);     /* for now */
1122                 return 0;
1123         }
1124         iterate_and_advance(i, bytes, v, ({
1125                 int err = 0;
1126                 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1127                                              v.iov_base,
1128                                              v.iov_len, 0, &err);
1129                 if (!err) {
1130                         sum = csum_block_add(sum, next, off);
1131                         off += v.iov_len;
1132                 }
1133                 err ? v.iov_len : 0;
1134         }), ({
1135                 char *p = kmap_atomic(v.bv_page);
1136                 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
1137                                                  p + v.bv_offset,
1138                                                  v.bv_len, 0);
1139                 kunmap_atomic(p);
1140                 sum = csum_block_add(sum, next, off);
1141                 off += v.bv_len;
1142         }),({
1143                 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
1144                                                  v.iov_base,
1145                                                  v.iov_len, 0);
1146                 sum = csum_block_add(sum, next, off);
1147                 off += v.iov_len;
1148         })
1149         )
1150         *csum = sum;
1151         return bytes;
1152 }
1153 EXPORT_SYMBOL(csum_and_copy_to_iter);
1154
1155 int iov_iter_npages(const struct iov_iter *i, int maxpages)
1156 {
1157         size_t size = i->count;
1158         int npages = 0;
1159
1160         if (!size)
1161                 return 0;
1162
1163         if (unlikely(i->type & ITER_PIPE)) {
1164                 struct pipe_inode_info *pipe = i->pipe;
1165                 size_t off;
1166                 int idx;
1167
1168                 if (!sanity(i))
1169                         return 0;
1170
1171                 data_start(i, &idx, &off);
1172                 /* some of this one + all after this one */
1173                 npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
1174                 if (npages >= maxpages)
1175                         return maxpages;
1176         } else iterate_all_kinds(i, size, v, ({
1177                 unsigned long p = (unsigned long)v.iov_base;
1178                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1179                         - p / PAGE_SIZE;
1180                 if (npages >= maxpages)
1181                         return maxpages;
1182         0;}),({
1183                 npages++;
1184                 if (npages >= maxpages)
1185                         return maxpages;
1186         }),({
1187                 unsigned long p = (unsigned long)v.iov_base;
1188                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1189                         - p / PAGE_SIZE;
1190                 if (npages >= maxpages)
1191                         return maxpages;
1192         })
1193         )
1194         return npages;
1195 }
1196 EXPORT_SYMBOL(iov_iter_npages);
1197
1198 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1199 {
1200         *new = *old;
1201         if (unlikely(new->type & ITER_PIPE)) {
1202                 WARN_ON(1);
1203                 return NULL;
1204         }
1205         if (new->type & ITER_BVEC)
1206                 return new->bvec = kmemdup(new->bvec,
1207                                     new->nr_segs * sizeof(struct bio_vec),
1208                                     flags);
1209         else
1210                 /* iovec and kvec have identical layout */
1211                 return new->iov = kmemdup(new->iov,
1212                                    new->nr_segs * sizeof(struct iovec),
1213                                    flags);
1214 }
1215 EXPORT_SYMBOL(dup_iter);
1216
1217 /**
1218  * import_iovec() - Copy an array of &struct iovec from userspace
1219  *     into the kernel, check that it is valid, and initialize a new
1220  *     &struct iov_iter iterator to access it.
1221  *
1222  * @type: One of %READ or %WRITE.
1223  * @uvector: Pointer to the userspace array.
1224  * @nr_segs: Number of elements in userspace array.
1225  * @fast_segs: Number of elements in @iov.
1226  * @iov: (input and output parameter) Pointer to pointer to (usually small
1227  *     on-stack) kernel array.
1228  * @i: Pointer to iterator that will be initialized on success.
1229  *
1230  * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1231  * then this function places %NULL in *@iov on return. Otherwise, a new
1232  * array will be allocated and the result placed in *@iov. This means that
1233  * the caller may call kfree() on *@iov regardless of whether the small
1234  * on-stack array was used or not (and regardless of whether this function
1235  * returns an error or not).
1236  *
1237  * Return: 0 on success or negative error code on error.
1238  */
1239 int import_iovec(int type, const struct iovec __user * uvector,
1240                  unsigned nr_segs, unsigned fast_segs,
1241                  struct iovec **iov, struct iov_iter *i)
1242 {
1243         ssize_t n;
1244         struct iovec *p;
1245         n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1246                                   *iov, &p);
1247         if (n < 0) {
1248                 if (p != *iov)
1249                         kfree(p);
1250                 *iov = NULL;
1251                 return n;
1252         }
1253         iov_iter_init(i, type, p, nr_segs, n);
1254         *iov = p == *iov ? NULL : p;
1255         return 0;
1256 }
1257 EXPORT_SYMBOL(import_iovec);
1258
1259 #ifdef CONFIG_COMPAT
1260 #include <linux/compat.h>
1261
1262 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
1263                  unsigned nr_segs, unsigned fast_segs,
1264                  struct iovec **iov, struct iov_iter *i)
1265 {
1266         ssize_t n;
1267         struct iovec *p;
1268         n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1269                                   *iov, &p);
1270         if (n < 0) {
1271                 if (p != *iov)
1272                         kfree(p);
1273                 *iov = NULL;
1274                 return n;
1275         }
1276         iov_iter_init(i, type, p, nr_segs, n);
1277         *iov = p == *iov ? NULL : p;
1278         return 0;
1279 }
1280 #endif
1281
1282 int import_single_range(int rw, void __user *buf, size_t len,
1283                  struct iovec *iov, struct iov_iter *i)
1284 {
1285         if (len > MAX_RW_COUNT)
1286                 len = MAX_RW_COUNT;
1287         if (unlikely(!access_ok(!rw, buf, len)))
1288                 return -EFAULT;
1289
1290         iov->iov_base = buf;
1291         iov->iov_len = len;
1292         iov_iter_init(i, rw, iov, 1, len);
1293         return 0;
1294 }
1295 EXPORT_SYMBOL(import_single_range);