net/sunrpc/xprtmultipath.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Multipath support for RPC
   4  *
   5  * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved.
   6  *
   7  * Trond Myklebust <trond.myklebust@primarydata.com>
   8  *
   9  */
  10 #include <linux/types.h>
  11 #include <linux/kref.h>
  12 #include <linux/list.h>
  13 #include <linux/rcupdate.h>
  14 #include <linux/rculist.h>
  15 #include <linux/slab.h>
  16 #include <asm/cmpxchg.h>
  17 #include <linux/spinlock.h>
  18 #include <linux/sunrpc/xprt.h>
  19 #include <linux/sunrpc/addr.h>
  20 #include <linux/sunrpc/xprtmultipath.h>
  21
  22 typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head,
  23                 const struct rpc_xprt *cur);
  24
  25 static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
  26 static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin;
  27 static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall;
  28
  29 static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
  30                 struct rpc_xprt *xprt)
  31 {
  32         if (unlikely(xprt_get(xprt) == NULL))
  33                 return;
  34         list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list);
  35         smp_wmb();
  36         if (xps->xps_nxprts == 0)
  37                 xps->xps_net = xprt->xprt_net;
  38         xps->xps_nxprts++;
  39 }
  40
  41 /**
  42  * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch
  43  * @xps: pointer to struct rpc_xprt_switch
  44  * @xprt: pointer to struct rpc_xprt
  45  *
  46  * Adds xprt to the end of the list of struct rpc_xprt in xps.
  47  */
  48 void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps,
  49                 struct rpc_xprt *xprt)
  50 {
  51         if (xprt == NULL)
  52                 return;
  53         spin_lock(&xps->xps_lock);
  54         if ((xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) &&
  55             !rpc_xprt_switch_has_addr(xps, (struct sockaddr *)&xprt->addr))
  56                 xprt_switch_add_xprt_locked(xps, xprt);
  57         spin_unlock(&xps->xps_lock);
  58 }
  59
  60 static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps,
  61                 struct rpc_xprt *xprt)
  62 {
  63         if (unlikely(xprt == NULL))
  64                 return;
  65         xps->xps_nxprts--;
  66         if (xps->xps_nxprts == 0)
  67                 xps->xps_net = NULL;
  68         smp_wmb();
  69         list_del_rcu(&xprt->xprt_switch);
  70 }
  71
  72 /**
  73  * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch
  74  * @xps: pointer to struct rpc_xprt_switch
  75  * @xprt: pointer to struct rpc_xprt
  76  *
  77  * Removes xprt from the list of struct rpc_xprt in xps.
  78  */
  79 void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps,
  80                 struct rpc_xprt *xprt)
  81 {
  82         spin_lock(&xps->xps_lock);
  83         xprt_switch_remove_xprt_locked(xps, xprt);
  84         spin_unlock(&xps->xps_lock);
  85         xprt_put(xprt);
  86 }
  87
  88 /**
  89  * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch
  90  * @xprt: pointer to struct rpc_xprt
  91  * @gfp_flags: allocation flags
  92  *
  93  * On success, returns an initialised struct rpc_xprt_switch, containing
  94  * the entry xprt. Returns NULL on failure.
  95  */
  96 struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt,
  97                 gfp_t gfp_flags)
  98 {
  99         struct rpc_xprt_switch *xps;
 100
 101         xps = kmalloc(sizeof(*xps), gfp_flags);
 102         if (xps != NULL) {
 103                 spin_lock_init(&xps->xps_lock);
 104                 kref_init(&xps->xps_kref);
 105                 xps->xps_nxprts = 0;
 106                 INIT_LIST_HEAD(&xps->xps_xprt_list);
 107                 xps->xps_iter_ops = &rpc_xprt_iter_singular;
 108                 xprt_switch_add_xprt_locked(xps, xprt);
 109         }
 110
 111         return xps;
 112 }
 113
 114 static void xprt_switch_free_entries(struct rpc_xprt_switch *xps)
 115 {
 116         spin_lock(&xps->xps_lock);
 117         while (!list_empty(&xps->xps_xprt_list)) {
 118                 struct rpc_xprt *xprt;
 119
 120                 xprt = list_first_entry(&xps->xps_xprt_list,
 121                                 struct rpc_xprt, xprt_switch);
 122                 xprt_switch_remove_xprt_locked(xps, xprt);
 123                 spin_unlock(&xps->xps_lock);
 124                 xprt_put(xprt);
 125                 spin_lock(&xps->xps_lock);
 126         }
 127         spin_unlock(&xps->xps_lock);
 128 }
 129
 130 static void xprt_switch_free(struct kref *kref)
 131 {
 132         struct rpc_xprt_switch *xps = container_of(kref,
 133                         struct rpc_xprt_switch, xps_kref);
 134
 135         xprt_switch_free_entries(xps);
 136         kfree_rcu(xps, xps_rcu);
 137 }
 138
 139 /**
 140  * xprt_switch_get - Return a reference to a rpc_xprt_switch
 141  * @xps: pointer to struct rpc_xprt_switch
 142  *
 143  * Returns a reference to xps unless the refcount is already zero.
 144  */
 145 struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps)
 146 {
 147         if (xps != NULL && kref_get_unless_zero(&xps->xps_kref))
 148                 return xps;
 149         return NULL;
 150 }
 151
 152 /**
 153  * xprt_switch_put - Release a reference to a rpc_xprt_switch
 154  * @xps: pointer to struct rpc_xprt_switch
 155  *
 156  * Release the reference to xps, and free it once the refcount is zero.
 157  */
 158 void xprt_switch_put(struct rpc_xprt_switch *xps)
 159 {
 160         if (xps != NULL)
 161                 kref_put(&xps->xps_kref, xprt_switch_free);
 162 }
 163
 164 /**
 165  * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch
 166  * @xps: pointer to struct rpc_xprt_switch
 167  *
 168  * Sets a round-robin default policy for iterators acting on xps.
 169  */
 170 void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps)
 171 {
 172         if (READ_ONCE(xps->xps_iter_ops) != &rpc_xprt_iter_roundrobin)
 173                 WRITE_ONCE(xps->xps_iter_ops, &rpc_xprt_iter_roundrobin);
 174 }
 175
 176 static
 177 const struct rpc_xprt_iter_ops *xprt_iter_ops(const struct rpc_xprt_iter *xpi)
 178 {
 179         if (xpi->xpi_ops != NULL)
 180                 return xpi->xpi_ops;
 181         return rcu_dereference(xpi->xpi_xpswitch)->xps_iter_ops;
 182 }
 183
 184 static
 185 void xprt_iter_no_rewind(struct rpc_xprt_iter *xpi)
 186 {
 187 }
 188
 189 static
 190 void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi)
 191 {
 192         WRITE_ONCE(xpi->xpi_cursor, NULL);
 193 }
 194
 195 static
 196 struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head)
 197 {
 198         return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch);
 199 }
 200
 201 static
 202 struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi)
 203 {
 204         struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
 205
 206         if (xps == NULL)
 207                 return NULL;
 208         return xprt_switch_find_first_entry(&xps->xps_xprt_list);
 209 }
 210
 211 static
 212 struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head,
 213                 const struct rpc_xprt *cur)
 214 {
 215         struct rpc_xprt *pos;
 216
 217         list_for_each_entry_rcu(pos, head, xprt_switch) {
 218                 if (cur == pos)
 219                         return pos;
 220         }
 221         return NULL;
 222 }
 223
 224 static
 225 struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi)
 226 {
 227         struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
 228         struct list_head *head;
 229
 230         if (xps == NULL)
 231                 return NULL;
 232         head = &xps->xps_xprt_list;
 233         if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2)
 234                 return xprt_switch_find_first_entry(head);
 235         return xprt_switch_find_current_entry(head, xpi->xpi_cursor);
 236 }
 237
 238 static
 239 bool __rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps,
 240                                 const struct sockaddr *sap)
 241 {
 242         struct list_head *head;
 243         struct rpc_xprt *pos;
 244
 245         if (xps == NULL || sap == NULL)
 246                 return false;
 247
 248         head = &xps->xps_xprt_list;
 249         list_for_each_entry_rcu(pos, head, xprt_switch) {
 250                 if (rpc_cmp_addr_port(sap, (struct sockaddr *)&pos->addr)) {
 251                         pr_info("RPC:   addr %s already in xprt switch\n",
 252                                 pos->address_strings[RPC_DISPLAY_ADDR]);
 253                         return true;
 254                 }
 255         }
 256         return false;
 257 }
 258
 259 bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps,
 260                               const struct sockaddr *sap)
 261 {
 262         bool res;
 263
 264         rcu_read_lock();
 265         res = __rpc_xprt_switch_has_addr(xps, sap);
 266         rcu_read_unlock();
 267
 268         return res;
 269 }
 270
 271 static
 272 struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head,
 273                 const struct rpc_xprt *cur)
 274 {
 275         struct rpc_xprt *pos, *prev = NULL;
 276
 277         list_for_each_entry_rcu(pos, head, xprt_switch) {
 278                 if (cur == prev)
 279                         return pos;
 280                 prev = pos;
 281         }
 282         return NULL;
 283 }
 284
 285 static
 286 struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head,
 287                 struct rpc_xprt **cursor,
 288                 xprt_switch_find_xprt_t find_next)
 289 {
 290         struct rpc_xprt *cur, *pos, *old;
 291
 292         cur = READ_ONCE(*cursor);
 293         for (;;) {
 294                 old = cur;
 295                 pos = find_next(head, old);
 296                 if (pos == NULL)
 297                         break;
 298                 cur = cmpxchg_relaxed(cursor, old, pos);
 299                 if (cur == old)
 300                         break;
 301         }
 302         return pos;
 303 }
 304
 305 static
 306 struct rpc_xprt *xprt_iter_next_entry_multiple(struct rpc_xprt_iter *xpi,
 307                 xprt_switch_find_xprt_t find_next)
 308 {
 309         struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
 310
 311         if (xps == NULL)
 312                 return NULL;
 313         return xprt_switch_set_next_cursor(&xps->xps_xprt_list,
 314                         &xpi->xpi_cursor,
 315                         find_next);
 316 }
 317
 318 static
 319 struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head,
 320                 const struct rpc_xprt *cur)
 321 {
 322         struct rpc_xprt *ret;
 323
 324         ret = xprt_switch_find_next_entry(head, cur);
 325         if (ret != NULL)
 326                 return ret;
 327         return xprt_switch_find_first_entry(head);
 328 }
 329
 330 static
 331 struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi)
 332 {
 333         return xprt_iter_next_entry_multiple(xpi,
 334                         xprt_switch_find_next_entry_roundrobin);
 335 }
 336
 337 static
 338 struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi)
 339 {
 340         return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry);
 341 }
 342
 343 /*
 344  * xprt_iter_rewind - Resets the xprt iterator
 345  * @xpi: pointer to rpc_xprt_iter
 346  *
 347  * Resets xpi to ensure that it points to the first entry in the list
 348  * of transports.
 349  */
 350 static
 351 void xprt_iter_rewind(struct rpc_xprt_iter *xpi)
 352 {
 353         rcu_read_lock();
 354         xprt_iter_ops(xpi)->xpi_rewind(xpi);
 355         rcu_read_unlock();
 356 }
 357
 358 static void __xprt_iter_init(struct rpc_xprt_iter *xpi,
 359                 struct rpc_xprt_switch *xps,
 360                 const struct rpc_xprt_iter_ops *ops)
 361 {
 362         rcu_assign_pointer(xpi->xpi_xpswitch, xprt_switch_get(xps));
 363         xpi->xpi_cursor = NULL;
 364         xpi->xpi_ops = ops;
 365 }
 366
 367 /**
 368  * xprt_iter_init - Initialise an xprt iterator
 369  * @xpi: pointer to rpc_xprt_iter
 370  * @xps: pointer to rpc_xprt_switch
 371  *
 372  * Initialises the iterator to use the default iterator ops
 373  * as set in xps. This function is mainly intended for internal
 374  * use in the rpc_client.
 375  */
 376 void xprt_iter_init(struct rpc_xprt_iter *xpi,
 377                 struct rpc_xprt_switch *xps)
 378 {
 379         __xprt_iter_init(xpi, xps, NULL);
 380 }
 381
 382 /**
 383  * xprt_iter_init_listall - Initialise an xprt iterator
 384  * @xpi: pointer to rpc_xprt_iter
 385  * @xps: pointer to rpc_xprt_switch
 386  *
 387  * Initialises the iterator to iterate once through the entire list
 388  * of entries in xps.
 389  */
 390 void xprt_iter_init_listall(struct rpc_xprt_iter *xpi,
 391                 struct rpc_xprt_switch *xps)
 392 {
 393         __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall);
 394 }
 395
 396 /**
 397  * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
 398  * @xpi: pointer to rpc_xprt_iter
 399  * @xps: pointer to a new rpc_xprt_switch or NULL
 400  *
 401  * Swaps out the existing xpi->xpi_xpswitch with a new value.
 402  */
 403 struct rpc_xprt_switch *xprt_iter_xchg_switch(struct rpc_xprt_iter *xpi,
 404                 struct rpc_xprt_switch *newswitch)
 405 {
 406         struct rpc_xprt_switch __rcu *oldswitch;
 407
 408         /* Atomically swap out the old xpswitch */
 409         oldswitch = xchg(&xpi->xpi_xpswitch, RCU_INITIALIZER(newswitch));
 410         if (newswitch != NULL)
 411                 xprt_iter_rewind(xpi);
 412         return rcu_dereference_protected(oldswitch, true);
 413 }
 414
 415 /**
 416  * xprt_iter_destroy - Destroys the xprt iterator
 417  * @xpi pointer to rpc_xprt_iter
 418  */
 419 void xprt_iter_destroy(struct rpc_xprt_iter *xpi)
 420 {
 421         xprt_switch_put(xprt_iter_xchg_switch(xpi, NULL));
 422 }
 423
 424 /**
 425  * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor
 426  * @xpi: pointer to rpc_xprt_iter
 427  *
 428  * Returns a pointer to the struct rpc_xprt that is currently
 429  * pointed to by the cursor.
 430  * Caller must be holding rcu_read_lock().
 431  */
 432 struct rpc_xprt *xprt_iter_xprt(struct rpc_xprt_iter *xpi)
 433 {
 434         WARN_ON_ONCE(!rcu_read_lock_held());
 435         return xprt_iter_ops(xpi)->xpi_xprt(xpi);
 436 }
 437
 438 static
 439 struct rpc_xprt *xprt_iter_get_helper(struct rpc_xprt_iter *xpi,
 440                 struct rpc_xprt *(*fn)(struct rpc_xprt_iter *))
 441 {
 442         struct rpc_xprt *ret;
 443
 444         do {
 445                 ret = fn(xpi);
 446                 if (ret == NULL)
 447                         break;
 448                 ret = xprt_get(ret);
 449         } while (ret == NULL);
 450         return ret;
 451 }
 452
 453 /**
 454  * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor
 455  * @xpi: pointer to rpc_xprt_iter
 456  *
 457  * Returns a reference to the struct rpc_xprt that is currently
 458  * pointed to by the cursor.
 459  */
 460 struct rpc_xprt *xprt_iter_get_xprt(struct rpc_xprt_iter *xpi)
 461 {
 462         struct rpc_xprt *xprt;
 463
 464         rcu_read_lock();
 465         xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_xprt);
 466         rcu_read_unlock();
 467         return xprt;
 468 }
 469
 470 /**
 471  * xprt_iter_get_next - Returns the next rpc_xprt following the cursor
 472  * @xpi: pointer to rpc_xprt_iter
 473  *
 474  * Returns a reference to the struct rpc_xprt that immediately follows the
 475  * entry pointed to by the cursor.
 476  */
 477 struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi)
 478 {
 479         struct rpc_xprt *xprt;
 480
 481         rcu_read_lock();
 482         xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_next);
 483         rcu_read_unlock();
 484         return xprt;
 485 }
 486
 487 /* Policy for always returning the first entry in the rpc_xprt_switch */
 488 static
 489 const struct rpc_xprt_iter_ops rpc_xprt_iter_singular = {
 490         .xpi_rewind = xprt_iter_no_rewind,
 491         .xpi_xprt = xprt_iter_first_entry,
 492         .xpi_next = xprt_iter_first_entry,
 493 };
 494
 495 /* Policy for round-robin iteration of entries in the rpc_xprt_switch */
 496 static
 497 const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin = {
 498         .xpi_rewind = xprt_iter_default_rewind,
 499         .xpi_xprt = xprt_iter_current_entry,
 500         .xpi_next = xprt_iter_next_entry_roundrobin,
 501 };
 502
 503 /* Policy for once-through iteration of entries in the rpc_xprt_switch */
 504 static
 505 const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = {
 506         .xpi_rewind = xprt_iter_default_rewind,
 507         .xpi_xprt = xprt_iter_current_entry,
 508         .xpi_next = xprt_iter_next_entry_all,
 509 };