net/rds/ib_rdma.c

   1 /*
   2  * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33 #include <linux/kernel.h>
  34 #include <linux/slab.h>
  35 #include <linux/rculist.h>
  36 #include <linux/llist.h>
  37
  38 #include "rds_single_path.h"
  39 #include "ib_mr.h"
  40
  41 struct workqueue_struct *rds_ib_mr_wq;
  42
  43 static DEFINE_PER_CPU(unsigned long, clean_list_grace);
  44 #define CLEAN_LIST_BUSY_BIT 0
  45
  46 static struct rds_ib_device *rds_ib_get_device(__be32 ipaddr)
  47 {
  48         struct rds_ib_device *rds_ibdev;
  49         struct rds_ib_ipaddr *i_ipaddr;
  50
  51         rcu_read_lock();
  52         list_for_each_entry_rcu(rds_ibdev, &rds_ib_devices, list) {
  53                 list_for_each_entry_rcu(i_ipaddr, &rds_ibdev->ipaddr_list, list) {
  54                         if (i_ipaddr->ipaddr == ipaddr) {
  55                                 refcount_inc(&rds_ibdev->refcount);
  56                                 rcu_read_unlock();
  57                                 return rds_ibdev;
  58                         }
  59                 }
  60         }
  61         rcu_read_unlock();
  62
  63         return NULL;
  64 }
  65
  66 static int rds_ib_add_ipaddr(struct rds_ib_device *rds_ibdev, __be32 ipaddr)
  67 {
  68         struct rds_ib_ipaddr *i_ipaddr;
  69
  70         i_ipaddr = kmalloc(sizeof *i_ipaddr, GFP_KERNEL);
  71         if (!i_ipaddr)
  72                 return -ENOMEM;
  73
  74         i_ipaddr->ipaddr = ipaddr;
  75
  76         spin_lock_irq(&rds_ibdev->spinlock);
  77         list_add_tail_rcu(&i_ipaddr->list, &rds_ibdev->ipaddr_list);
  78         spin_unlock_irq(&rds_ibdev->spinlock);
  79
  80         return 0;
  81 }
  82
  83 static void rds_ib_remove_ipaddr(struct rds_ib_device *rds_ibdev, __be32 ipaddr)
  84 {
  85         struct rds_ib_ipaddr *i_ipaddr;
  86         struct rds_ib_ipaddr *to_free = NULL;
  87
  88
  89         spin_lock_irq(&rds_ibdev->spinlock);
  90         list_for_each_entry_rcu(i_ipaddr, &rds_ibdev->ipaddr_list, list) {
  91                 if (i_ipaddr->ipaddr == ipaddr) {
  92                         list_del_rcu(&i_ipaddr->list);
  93                         to_free = i_ipaddr;
  94                         break;
  95                 }
  96         }
  97         spin_unlock_irq(&rds_ibdev->spinlock);
  98
  99         if (to_free)
 100                 kfree_rcu(to_free, rcu);
 101 }
 102
 103 int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev,
 104                          struct in6_addr *ipaddr)
 105 {
 106         struct rds_ib_device *rds_ibdev_old;
 107
 108         rds_ibdev_old = rds_ib_get_device(ipaddr->s6_addr32[3]);
 109         if (!rds_ibdev_old)
 110                 return rds_ib_add_ipaddr(rds_ibdev, ipaddr->s6_addr32[3]);
 111
 112         if (rds_ibdev_old != rds_ibdev) {
 113                 rds_ib_remove_ipaddr(rds_ibdev_old, ipaddr->s6_addr32[3]);
 114                 rds_ib_dev_put(rds_ibdev_old);
 115                 return rds_ib_add_ipaddr(rds_ibdev, ipaddr->s6_addr32[3]);
 116         }
 117         rds_ib_dev_put(rds_ibdev_old);
 118
 119         return 0;
 120 }
 121
 122 void rds_ib_add_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn)
 123 {
 124         struct rds_ib_connection *ic = conn->c_transport_data;
 125
 126         /* conn was previously on the nodev_conns_list */
 127         spin_lock_irq(&ib_nodev_conns_lock);
 128         BUG_ON(list_empty(&ib_nodev_conns));
 129         BUG_ON(list_empty(&ic->ib_node));
 130         list_del(&ic->ib_node);
 131
 132         spin_lock(&rds_ibdev->spinlock);
 133         list_add_tail(&ic->ib_node, &rds_ibdev->conn_list);
 134         spin_unlock(&rds_ibdev->spinlock);
 135         spin_unlock_irq(&ib_nodev_conns_lock);
 136
 137         ic->rds_ibdev = rds_ibdev;
 138         refcount_inc(&rds_ibdev->refcount);
 139 }
 140
 141 void rds_ib_remove_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn)
 142 {
 143         struct rds_ib_connection *ic = conn->c_transport_data;
 144
 145         /* place conn on nodev_conns_list */
 146         spin_lock(&ib_nodev_conns_lock);
 147
 148         spin_lock_irq(&rds_ibdev->spinlock);
 149         BUG_ON(list_empty(&ic->ib_node));
 150         list_del(&ic->ib_node);
 151         spin_unlock_irq(&rds_ibdev->spinlock);
 152
 153         list_add_tail(&ic->ib_node, &ib_nodev_conns);
 154
 155         spin_unlock(&ib_nodev_conns_lock);
 156
 157         ic->rds_ibdev = NULL;
 158         rds_ib_dev_put(rds_ibdev);
 159 }
 160
 161 void rds_ib_destroy_nodev_conns(void)
 162 {
 163         struct rds_ib_connection *ic, *_ic;
 164         LIST_HEAD(tmp_list);
 165
 166         /* avoid calling conn_destroy with irqs off */
 167         spin_lock_irq(&ib_nodev_conns_lock);
 168         list_splice(&ib_nodev_conns, &tmp_list);
 169         spin_unlock_irq(&ib_nodev_conns_lock);
 170
 171         list_for_each_entry_safe(ic, _ic, &tmp_list, ib_node)
 172                 rds_conn_destroy(ic->conn);
 173 }
 174
 175 void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev, struct rds_info_rdma_connection *iinfo)
 176 {
 177         struct rds_ib_mr_pool *pool_1m = rds_ibdev->mr_1m_pool;
 178
 179         iinfo->rdma_mr_max = pool_1m->max_items;
 180         iinfo->rdma_mr_size = pool_1m->fmr_attr.max_pages;
 181 }
 182
 183 #if IS_ENABLED(CONFIG_IPV6)
 184 void rds6_ib_get_mr_info(struct rds_ib_device *rds_ibdev,
 185                          struct rds6_info_rdma_connection *iinfo6)
 186 {
 187         struct rds_ib_mr_pool *pool_1m = rds_ibdev->mr_1m_pool;
 188
 189         iinfo6->rdma_mr_max = pool_1m->max_items;
 190         iinfo6->rdma_mr_size = pool_1m->fmr_attr.max_pages;
 191 }
 192 #endif
 193
 194 struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *pool)
 195 {
 196         struct rds_ib_mr *ibmr = NULL;
 197         struct llist_node *ret;
 198         unsigned long *flag;
 199
 200         preempt_disable();
 201         flag = this_cpu_ptr(&clean_list_grace);
 202         set_bit(CLEAN_LIST_BUSY_BIT, flag);
 203         ret = llist_del_first(&pool->clean_list);
 204         if (ret) {
 205                 ibmr = llist_entry(ret, struct rds_ib_mr, llnode);
 206                 if (pool->pool_type == RDS_IB_MR_8K_POOL)
 207                         rds_ib_stats_inc(s_ib_rdma_mr_8k_reused);
 208                 else
 209                         rds_ib_stats_inc(s_ib_rdma_mr_1m_reused);
 210         }
 211
 212         clear_bit(CLEAN_LIST_BUSY_BIT, flag);
 213         preempt_enable();
 214         return ibmr;
 215 }
 216
 217 static inline void wait_clean_list_grace(void)
 218 {
 219         int cpu;
 220         unsigned long *flag;
 221
 222         for_each_online_cpu(cpu) {
 223                 flag = &per_cpu(clean_list_grace, cpu);
 224                 while (test_bit(CLEAN_LIST_BUSY_BIT, flag))
 225                         cpu_relax();
 226         }
 227 }
 228
 229 void rds_ib_sync_mr(void *trans_private, int direction)
 230 {
 231         struct rds_ib_mr *ibmr = trans_private;
 232         struct rds_ib_device *rds_ibdev = ibmr->device;
 233
 234         switch (direction) {
 235         case DMA_FROM_DEVICE:
 236                 ib_dma_sync_sg_for_cpu(rds_ibdev->dev, ibmr->sg,
 237                         ibmr->sg_dma_len, DMA_BIDIRECTIONAL);
 238                 break;
 239         case DMA_TO_DEVICE:
 240                 ib_dma_sync_sg_for_device(rds_ibdev->dev, ibmr->sg,
 241                         ibmr->sg_dma_len, DMA_BIDIRECTIONAL);
 242                 break;
 243         }
 244 }
 245
 246 void __rds_ib_teardown_mr(struct rds_ib_mr *ibmr)
 247 {
 248         struct rds_ib_device *rds_ibdev = ibmr->device;
 249
 250         if (ibmr->sg_dma_len) {
 251                 ib_dma_unmap_sg(rds_ibdev->dev,
 252                                 ibmr->sg, ibmr->sg_len,
 253                                 DMA_BIDIRECTIONAL);
 254                 ibmr->sg_dma_len = 0;
 255         }
 256
 257         /* Release the s/g list */
 258         if (ibmr->sg_len) {
 259                 unsigned int i;
 260
 261                 for (i = 0; i < ibmr->sg_len; ++i) {
 262                         struct page *page = sg_page(&ibmr->sg[i]);
 263
 264                         /* FIXME we need a way to tell a r/w MR
 265                          * from a r/o MR */
 266                         WARN_ON(!page->mapping && irqs_disabled());
 267                         set_page_dirty(page);
 268                         put_page(page);
 269                 }
 270                 kfree(ibmr->sg);
 271
 272                 ibmr->sg = NULL;
 273                 ibmr->sg_len = 0;
 274         }
 275 }
 276
 277 void rds_ib_teardown_mr(struct rds_ib_mr *ibmr)
 278 {
 279         unsigned int pinned = ibmr->sg_len;
 280
 281         __rds_ib_teardown_mr(ibmr);
 282         if (pinned) {
 283                 struct rds_ib_mr_pool *pool = ibmr->pool;
 284
 285                 atomic_sub(pinned, &pool->free_pinned);
 286         }
 287 }
 288
 289 static inline unsigned int rds_ib_flush_goal(struct rds_ib_mr_pool *pool, int free_all)
 290 {
 291         unsigned int item_count;
 292
 293         item_count = atomic_read(&pool->item_count);
 294         if (free_all)
 295                 return item_count;
 296
 297         return 0;
 298 }
 299
 300 /*
 301  * given an llist of mrs, put them all into the list_head for more processing
 302  */
 303 static unsigned int llist_append_to_list(struct llist_head *llist,
 304                                          struct list_head *list)
 305 {
 306         struct rds_ib_mr *ibmr;
 307         struct llist_node *node;
 308         struct llist_node *next;
 309         unsigned int count = 0;
 310
 311         node = llist_del_all(llist);
 312         while (node) {
 313                 next = node->next;
 314                 ibmr = llist_entry(node, struct rds_ib_mr, llnode);
 315                 list_add_tail(&ibmr->unmap_list, list);
 316                 node = next;
 317                 count++;
 318         }
 319         return count;
 320 }
 321
 322 /*
 323  * this takes a list head of mrs and turns it into linked llist nodes
 324  * of clusters.  Each cluster has linked llist nodes of
 325  * MR_CLUSTER_SIZE mrs that are ready for reuse.
 326  */
 327 static void list_to_llist_nodes(struct rds_ib_mr_pool *pool,
 328                                 struct list_head *list,
 329                                 struct llist_node **nodes_head,
 330                                 struct llist_node **nodes_tail)
 331 {
 332         struct rds_ib_mr *ibmr;
 333         struct llist_node *cur = NULL;
 334         struct llist_node **next = nodes_head;
 335
 336         list_for_each_entry(ibmr, list, unmap_list) {
 337                 cur = &ibmr->llnode;
 338                 *next = cur;
 339                 next = &cur->next;
 340         }
 341         *next = NULL;
 342         *nodes_tail = cur;
 343 }
 344
 345 /*
 346  * Flush our pool of MRs.
 347  * At a minimum, all currently unused MRs are unmapped.
 348  * If the number of MRs allocated exceeds the limit, we also try
 349  * to free as many MRs as needed to get back to this limit.
 350  */
 351 int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,
 352                          int free_all, struct rds_ib_mr **ibmr_ret)
 353 {
 354         struct rds_ib_mr *ibmr;
 355         struct llist_node *clean_nodes;
 356         struct llist_node *clean_tail;
 357         LIST_HEAD(unmap_list);
 358         unsigned long unpinned = 0;
 359         unsigned int nfreed = 0, dirty_to_clean = 0, free_goal;
 360
 361         if (pool->pool_type == RDS_IB_MR_8K_POOL)
 362                 rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_flush);
 363         else
 364                 rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_flush);
 365
 366         if (ibmr_ret) {
 367                 DEFINE_WAIT(wait);
 368                 while (!mutex_trylock(&pool->flush_lock)) {
 369                         ibmr = rds_ib_reuse_mr(pool);
 370                         if (ibmr) {
 371                                 *ibmr_ret = ibmr;
 372                                 finish_wait(&pool->flush_wait, &wait);
 373                                 goto out_nolock;
 374                         }
 375
 376                         prepare_to_wait(&pool->flush_wait, &wait,
 377                                         TASK_UNINTERRUPTIBLE);
 378                         if (llist_empty(&pool->clean_list))
 379                                 schedule();
 380
 381                         ibmr = rds_ib_reuse_mr(pool);
 382                         if (ibmr) {
 383                                 *ibmr_ret = ibmr;
 384                                 finish_wait(&pool->flush_wait, &wait);
 385                                 goto out_nolock;
 386                         }
 387                 }
 388                 finish_wait(&pool->flush_wait, &wait);
 389         } else
 390                 mutex_lock(&pool->flush_lock);
 391
 392         if (ibmr_ret) {
 393                 ibmr = rds_ib_reuse_mr(pool);
 394                 if (ibmr) {
 395                         *ibmr_ret = ibmr;
 396                         goto out;
 397                 }
 398         }
 399
 400         /* Get the list of all MRs to be dropped. Ordering matters -
 401          * we want to put drop_list ahead of free_list.
 402          */
 403         dirty_to_clean = llist_append_to_list(&pool->drop_list, &unmap_list);
 404         dirty_to_clean += llist_append_to_list(&pool->free_list, &unmap_list);
 405         if (free_all)
 406                 llist_append_to_list(&pool->clean_list, &unmap_list);
 407
 408         free_goal = rds_ib_flush_goal(pool, free_all);
 409
 410         if (list_empty(&unmap_list))
 411                 goto out;
 412
 413         if (pool->use_fastreg)
 414                 rds_ib_unreg_frmr(&unmap_list, &nfreed, &unpinned, free_goal);
 415         else
 416                 rds_ib_unreg_fmr(&unmap_list, &nfreed, &unpinned, free_goal);
 417
 418         if (!list_empty(&unmap_list)) {
 419                 /* we have to make sure that none of the things we're about
 420                  * to put on the clean list would race with other cpus trying
 421                  * to pull items off.  The llist would explode if we managed to
 422                  * remove something from the clean list and then add it back again
 423                  * while another CPU was spinning on that same item in llist_del_first.
 424                  *
 425                  * This is pretty unlikely, but just in case  wait for an llist grace period
 426                  * here before adding anything back into the clean list.
 427                  */
 428                 wait_clean_list_grace();
 429
 430                 list_to_llist_nodes(pool, &unmap_list, &clean_nodes, &clean_tail);
 431                 if (ibmr_ret) {
 432                         *ibmr_ret = llist_entry(clean_nodes, struct rds_ib_mr, llnode);
 433                         clean_nodes = clean_nodes->next;
 434                 }
 435                 /* more than one entry in llist nodes */
 436                 if (clean_nodes)
 437                         llist_add_batch(clean_nodes, clean_tail,
 438                                         &pool->clean_list);
 439
 440         }
 441
 442         atomic_sub(unpinned, &pool->free_pinned);
 443         atomic_sub(dirty_to_clean, &pool->dirty_count);
 444         atomic_sub(nfreed, &pool->item_count);
 445
 446 out:
 447         mutex_unlock(&pool->flush_lock);
 448         if (waitqueue_active(&pool->flush_wait))
 449                 wake_up(&pool->flush_wait);
 450 out_nolock:
 451         return 0;
 452 }
 453
 454 struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *pool)
 455 {
 456         struct rds_ib_mr *ibmr = NULL;
 457         int iter = 0;
 458
 459         while (1) {
 460                 ibmr = rds_ib_reuse_mr(pool);
 461                 if (ibmr)
 462                         return ibmr;
 463
 464                 if (atomic_inc_return(&pool->item_count) <= pool->max_items)
 465                         break;
 466
 467                 atomic_dec(&pool->item_count);
 468
 469                 if (++iter > 2) {
 470                         if (pool->pool_type == RDS_IB_MR_8K_POOL)
 471                                 rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_depleted);
 472                         else
 473                                 rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_depleted);
 474                         return ERR_PTR(-EAGAIN);
 475                 }
 476
 477                 /* We do have some empty MRs. Flush them out. */
 478                 if (pool->pool_type == RDS_IB_MR_8K_POOL)
 479                         rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_wait);
 480                 else
 481                         rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_wait);
 482
 483                 rds_ib_flush_mr_pool(pool, 0, &ibmr);
 484                 if (ibmr)
 485                         return ibmr;
 486         }
 487
 488         return ibmr;
 489 }
 490
 491 static void rds_ib_mr_pool_flush_worker(struct work_struct *work)
 492 {
 493         struct rds_ib_mr_pool *pool = container_of(work, struct rds_ib_mr_pool, flush_worker.work);
 494
 495         rds_ib_flush_mr_pool(pool, 0, NULL);
 496 }
 497
 498 void rds_ib_free_mr(void *trans_private, int invalidate)
 499 {
 500         struct rds_ib_mr *ibmr = trans_private;
 501         struct rds_ib_mr_pool *pool = ibmr->pool;
 502         struct rds_ib_device *rds_ibdev = ibmr->device;
 503
 504         rdsdebug("RDS/IB: free_mr nents %u\n", ibmr->sg_len);
 505
 506         /* Return it to the pool's free list */
 507         if (rds_ibdev->use_fastreg)
 508                 rds_ib_free_frmr_list(ibmr);
 509         else
 510                 rds_ib_free_fmr_list(ibmr);
 511
 512         atomic_add(ibmr->sg_len, &pool->free_pinned);
 513         atomic_inc(&pool->dirty_count);
 514
 515         /* If we've pinned too many pages, request a flush */
 516         if (atomic_read(&pool->free_pinned) >= pool->max_free_pinned ||
 517             atomic_read(&pool->dirty_count) >= pool->max_items / 5)
 518                 queue_delayed_work(rds_ib_mr_wq, &pool->flush_worker, 10);
 519
 520         if (invalidate) {
 521                 if (likely(!in_interrupt())) {
 522                         rds_ib_flush_mr_pool(pool, 0, NULL);
 523                 } else {
 524                         /* We get here if the user created a MR marked
 525                          * as use_once and invalidate at the same time.
 526                          */
 527                         queue_delayed_work(rds_ib_mr_wq,
 528                                            &pool->flush_worker, 10);
 529                 }
 530         }
 531
 532         rds_ib_dev_put(rds_ibdev);
 533 }
 534
 535 void rds_ib_flush_mrs(void)
 536 {
 537         struct rds_ib_device *rds_ibdev;
 538
 539         down_read(&rds_ib_devices_lock);
 540         list_for_each_entry(rds_ibdev, &rds_ib_devices, list) {
 541                 if (rds_ibdev->mr_8k_pool)
 542                         rds_ib_flush_mr_pool(rds_ibdev->mr_8k_pool, 0, NULL);
 543
 544                 if (rds_ibdev->mr_1m_pool)
 545                         rds_ib_flush_mr_pool(rds_ibdev->mr_1m_pool, 0, NULL);
 546         }
 547         up_read(&rds_ib_devices_lock);
 548 }
 549
 550 void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents,
 551                     struct rds_sock *rs, u32 *key_ret,
 552                     struct rds_connection *conn)
 553 {
 554         struct rds_ib_device *rds_ibdev;
 555         struct rds_ib_mr *ibmr = NULL;
 556         struct rds_ib_connection *ic = NULL;
 557         int ret;
 558
 559         rds_ibdev = rds_ib_get_device(rs->rs_bound_addr.s6_addr32[3]);
 560         if (!rds_ibdev) {
 561                 ret = -ENODEV;
 562                 goto out;
 563         }
 564
 565         if (conn)
 566                 ic = conn->c_transport_data;
 567
 568         if (!rds_ibdev->mr_8k_pool || !rds_ibdev->mr_1m_pool) {
 569                 ret = -ENODEV;
 570                 goto out;
 571         }
 572
 573         if (rds_ibdev->use_fastreg)
 574                 ibmr = rds_ib_reg_frmr(rds_ibdev, ic, sg, nents, key_ret);
 575         else
 576                 ibmr = rds_ib_reg_fmr(rds_ibdev, sg, nents, key_ret);
 577         if (IS_ERR(ibmr)) {
 578                 ret = PTR_ERR(ibmr);
 579                 pr_warn("RDS/IB: rds_ib_get_mr failed (errno=%d)\n", ret);
 580         } else {
 581                 return ibmr;
 582         }
 583
 584  out:
 585         if (rds_ibdev)
 586                 rds_ib_dev_put(rds_ibdev);
 587
 588         return ERR_PTR(ret);
 589 }
 590
 591 void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *pool)
 592 {
 593         cancel_delayed_work_sync(&pool->flush_worker);
 594         rds_ib_flush_mr_pool(pool, 1, NULL);
 595         WARN_ON(atomic_read(&pool->item_count));
 596         WARN_ON(atomic_read(&pool->free_pinned));
 597         kfree(pool);
 598 }
 599
 600 struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_ibdev,
 601                                              int pool_type)
 602 {
 603         struct rds_ib_mr_pool *pool;
 604
 605         pool = kzalloc(sizeof(*pool), GFP_KERNEL);
 606         if (!pool)
 607                 return ERR_PTR(-ENOMEM);
 608
 609         pool->pool_type = pool_type;
 610         init_llist_head(&pool->free_list);
 611         init_llist_head(&pool->drop_list);
 612         init_llist_head(&pool->clean_list);
 613         mutex_init(&pool->flush_lock);
 614         init_waitqueue_head(&pool->flush_wait);
 615         INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker);
 616
 617         if (pool_type == RDS_IB_MR_1M_POOL) {
 618                 /* +1 allows for unaligned MRs */
 619                 pool->fmr_attr.max_pages = RDS_MR_1M_MSG_SIZE + 1;
 620                 pool->max_items = rds_ibdev->max_1m_mrs;
 621         } else {
 622                 /* pool_type == RDS_IB_MR_8K_POOL */
 623                 pool->fmr_attr.max_pages = RDS_MR_8K_MSG_SIZE + 1;
 624                 pool->max_items = rds_ibdev->max_8k_mrs;
 625         }
 626
 627         pool->max_free_pinned = pool->max_items * pool->fmr_attr.max_pages / 4;
 628         pool->fmr_attr.max_maps = rds_ibdev->fmr_max_remaps;
 629         pool->fmr_attr.page_shift = PAGE_SHIFT;
 630         pool->max_items_soft = rds_ibdev->max_mrs * 3 / 4;
 631         pool->use_fastreg = rds_ibdev->use_fastreg;
 632
 633         return pool;
 634 }
 635
 636 int rds_ib_mr_init(void)
 637 {
 638         rds_ib_mr_wq = alloc_workqueue("rds_mr_flushd", WQ_MEM_RECLAIM, 0);
 639         if (!rds_ib_mr_wq)
 640                 return -ENOMEM;
 641         return 0;
 642 }
 643
 644 /* By the time this is called all the IB devices should have been torn down and
 645  * had their pools freed.  As each pool is freed its work struct is waited on,
 646  * so the pool flushing work queue should be idle by the time we get here.
 647  */
 648 void rds_ib_mr_exit(void)
 649 {
 650         destroy_workqueue(rds_ib_mr_wq);
 651 }