fs/afs/vl_rotate.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* Handle vlserver selection and rotation.
   3  *
   4  * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  */
   7
   8 #include <linux/kernel.h>
   9 #include <linux/sched.h>
  10 #include <linux/sched/signal.h>
  11 #include "internal.h"
  12 #include "afs_vl.h"
  13
  14 /*
  15  * Begin an operation on a volume location server.
  16  */
  17 bool afs_begin_vlserver_operation(struct afs_vl_cursor *vc, struct afs_cell *cell,
  18                                   struct key *key)
  19 {
  20         memset(vc, 0, sizeof(*vc));
  21         vc->cell = cell;
  22         vc->key = key;
  23         vc->cumul_error.error = -EDESTADDRREQ;
  24         vc->nr_iterations = -1;
  25
  26         if (signal_pending(current)) {
  27                 vc->cumul_error.error = -EINTR;
  28                 vc->flags |= AFS_VL_CURSOR_STOP;
  29                 return false;
  30         }
  31
  32         return true;
  33 }
  34
  35 /*
  36  * Begin iteration through a server list, starting with the last used server if
  37  * possible, or the last recorded good server if not.
  38  */
  39 static bool afs_start_vl_iteration(struct afs_vl_cursor *vc)
  40 {
  41         struct afs_cell *cell = vc->cell;
  42         unsigned int dns_lookup_count;
  43
  44         if (cell->dns_source == DNS_RECORD_UNAVAILABLE ||
  45             cell->dns_expiry <= ktime_get_real_seconds()) {
  46                 dns_lookup_count = smp_load_acquire(&cell->dns_lookup_count);
  47                 set_bit(AFS_CELL_FL_DO_LOOKUP, &cell->flags);
  48                 afs_queue_cell(cell, afs_cell_trace_get_queue_dns);
  49
  50                 if (cell->dns_source == DNS_RECORD_UNAVAILABLE) {
  51                         if (wait_var_event_interruptible(
  52                                     &cell->dns_lookup_count,
  53                                     smp_load_acquire(&cell->dns_lookup_count)
  54                                     != dns_lookup_count) < 0) {
  55                                 vc->cumul_error.error = -ERESTARTSYS;
  56                                 return false;
  57                         }
  58                 }
  59
  60                 /* Status load is ordered after lookup counter load */
  61                 if (cell->dns_status == DNS_LOOKUP_GOT_NOT_FOUND) {
  62                         pr_warn("No record of cell %s\n", cell->name);
  63                         vc->cumul_error.error = -ENOENT;
  64                         return false;
  65                 }
  66
  67                 if (cell->dns_source == DNS_RECORD_UNAVAILABLE) {
  68                         vc->cumul_error.error = -EDESTADDRREQ;
  69                         return false;
  70                 }
  71         }
  72
  73         read_lock(&cell->vl_servers_lock);
  74         vc->server_list = afs_get_vlserverlist(
  75                 rcu_dereference_protected(cell->vl_servers,
  76                                           lockdep_is_held(&cell->vl_servers_lock)));
  77         read_unlock(&cell->vl_servers_lock);
  78         if (!vc->server_list->nr_servers)
  79                 return false;
  80
  81         vc->untried = (1UL << vc->server_list->nr_servers) - 1;
  82         vc->index = -1;
  83         return true;
  84 }
  85
  86 /*
  87  * Select the vlserver to use.  May be called multiple times to rotate
  88  * through the vlservers.
  89  */
  90 bool afs_select_vlserver(struct afs_vl_cursor *vc)
  91 {
  92         struct afs_addr_list *alist;
  93         struct afs_vlserver *vlserver;
  94         unsigned int rtt;
  95         s32 abort_code = vc->call_abort_code;
  96         int error = vc->call_error, i;
  97
  98         vc->nr_iterations++;
  99
 100         _enter("%lx[%d],%lx[%d],%d,%d",
 101                vc->untried, vc->index,
 102                vc->ac.tried, vc->ac.index,
 103                error, abort_code);
 104
 105         if (vc->flags & AFS_VL_CURSOR_STOP) {
 106                 _leave(" = f [stopped]");
 107                 return false;
 108         }
 109
 110         if (vc->nr_iterations == 0)
 111                 goto start;
 112
 113         /* Evaluate the result of the previous operation, if there was one. */
 114         switch (error) {
 115         default:
 116         case 0:
 117                 /* Success or local failure.  Stop. */
 118                 vc->cumul_error.error = error;
 119                 vc->flags |= AFS_VL_CURSOR_STOP;
 120                 _leave(" = f [okay/local %d]", vc->cumul_error.error);
 121                 return false;
 122
 123         case -ECONNABORTED:
 124                 /* The far side rejected the operation on some grounds.  This
 125                  * might involve the server being busy or the volume having been moved.
 126                  */
 127                 switch (abort_code) {
 128                 case AFSVL_IO:
 129                 case AFSVL_BADVOLOPER:
 130                 case AFSVL_NOMEM:
 131                         /* The server went weird. */
 132                         afs_prioritise_error(&vc->cumul_error, -EREMOTEIO, abort_code);
 133                         //write_lock(&vc->cell->vl_servers_lock);
 134                         //vc->server_list->weird_mask |= 1 << vc->index;
 135                         //write_unlock(&vc->cell->vl_servers_lock);
 136                         goto next_server;
 137
 138                 default:
 139                         afs_prioritise_error(&vc->cumul_error, error, abort_code);
 140                         goto failed;
 141                 }
 142
 143         case -ERFKILL:
 144         case -EADDRNOTAVAIL:
 145         case -ENETUNREACH:
 146         case -EHOSTUNREACH:
 147         case -EHOSTDOWN:
 148         case -ECONNREFUSED:
 149         case -ETIMEDOUT:
 150         case -ETIME:
 151                 _debug("no conn %d", error);
 152                 afs_prioritise_error(&vc->cumul_error, error, 0);
 153                 goto iterate_address;
 154
 155         case -ECONNRESET:
 156                 _debug("call reset");
 157                 afs_prioritise_error(&vc->cumul_error, error, 0);
 158                 vc->flags |= AFS_VL_CURSOR_RETRY;
 159                 goto next_server;
 160
 161         case -EOPNOTSUPP:
 162                 _debug("notsupp");
 163                 goto next_server;
 164         }
 165
 166 restart_from_beginning:
 167         _debug("restart");
 168         afs_end_cursor(&vc->ac);
 169         afs_put_vlserverlist(vc->cell->net, vc->server_list);
 170         vc->server_list = NULL;
 171         if (vc->flags & AFS_VL_CURSOR_RETRIED)
 172                 goto failed;
 173         vc->flags |= AFS_VL_CURSOR_RETRIED;
 174 start:
 175         _debug("start");
 176
 177         if (!afs_start_vl_iteration(vc))
 178                 goto failed;
 179
 180         error = afs_send_vl_probes(vc->cell->net, vc->key, vc->server_list);
 181         if (error < 0) {
 182                 afs_prioritise_error(&vc->cumul_error, error, 0);
 183                 goto failed;
 184         }
 185
 186 pick_server:
 187         _debug("pick [%lx]", vc->untried);
 188
 189         error = afs_wait_for_vl_probes(vc->server_list, vc->untried);
 190         if (error < 0) {
 191                 afs_prioritise_error(&vc->cumul_error, error, 0);
 192                 goto failed;
 193         }
 194
 195         /* Pick the untried server with the lowest RTT. */
 196         vc->index = vc->server_list->preferred;
 197         if (test_bit(vc->index, &vc->untried))
 198                 goto selected_server;
 199
 200         vc->index = -1;
 201         rtt = UINT_MAX;
 202         for (i = 0; i < vc->server_list->nr_servers; i++) {
 203                 struct afs_vlserver *s = vc->server_list->servers[i].server;
 204
 205                 if (!test_bit(i, &vc->untried) ||
 206                     !test_bit(AFS_VLSERVER_FL_RESPONDING, &s->flags))
 207                         continue;
 208                 if (s->probe.rtt < rtt) {
 209                         vc->index = i;
 210                         rtt = s->probe.rtt;
 211                 }
 212         }
 213
 214         if (vc->index == -1)
 215                 goto no_more_servers;
 216
 217 selected_server:
 218         _debug("use %d", vc->index);
 219         __clear_bit(vc->index, &vc->untried);
 220
 221         /* We're starting on a different vlserver from the list.  We need to
 222          * check it, find its address list and probe its capabilities before we
 223          * use it.
 224          */
 225         ASSERTCMP(vc->ac.alist, ==, NULL);
 226         vlserver = vc->server_list->servers[vc->index].server;
 227         vc->server = vlserver;
 228
 229         _debug("USING VLSERVER: %s", vlserver->name);
 230
 231         read_lock(&vlserver->lock);
 232         alist = rcu_dereference_protected(vlserver->addresses,
 233                                           lockdep_is_held(&vlserver->lock));
 234         afs_get_addrlist(alist);
 235         read_unlock(&vlserver->lock);
 236
 237         memset(&vc->ac, 0, sizeof(vc->ac));
 238
 239         if (!vc->ac.alist)
 240                 vc->ac.alist = alist;
 241         else
 242                 afs_put_addrlist(alist);
 243
 244         vc->ac.index = -1;
 245
 246 iterate_address:
 247         ASSERT(vc->ac.alist);
 248         /* Iterate over the current server's address list to try and find an
 249          * address on which it will respond to us.
 250          */
 251         if (!afs_iterate_addresses(&vc->ac))
 252                 goto next_server;
 253
 254         _debug("VL address %d/%d", vc->ac.index, vc->ac.alist->nr_addrs);
 255
 256         vc->call_responded = false;
 257         _leave(" = t %pISpc", rxrpc_kernel_remote_addr(vc->ac.alist->addrs[vc->ac.index].peer));
 258         return true;
 259
 260 next_server:
 261         _debug("next");
 262         afs_end_cursor(&vc->ac);
 263         goto pick_server;
 264
 265 no_more_servers:
 266         /* That's all the servers poked to no good effect.  Try again if some
 267          * of them were busy.
 268          */
 269         if (vc->flags & AFS_VL_CURSOR_RETRY)
 270                 goto restart_from_beginning;
 271
 272         for (i = 0; i < vc->server_list->nr_servers; i++) {
 273                 struct afs_vlserver *s = vc->server_list->servers[i].server;
 274
 275                 if (test_bit(AFS_VLSERVER_FL_RESPONDING, &s->flags))
 276                         vc->cumul_error.responded = true;
 277                 afs_prioritise_error(&vc->cumul_error, READ_ONCE(s->probe.error),
 278                                      s->probe.abort_code);
 279         }
 280
 281 failed:
 282         vc->flags |= AFS_VL_CURSOR_STOP;
 283         afs_end_cursor(&vc->ac);
 284         _leave(" = f [failed %d]", vc->cumul_error.error);
 285         return false;
 286 }
 287
 288 /*
 289  * Dump cursor state in the case of the error being EDESTADDRREQ.
 290  */
 291 static void afs_vl_dump_edestaddrreq(const struct afs_vl_cursor *vc)
 292 {
 293         struct afs_cell *cell = vc->cell;
 294         static int count;
 295         int i;
 296
 297         if (!IS_ENABLED(CONFIG_AFS_DEBUG_CURSOR) || count > 3)
 298                 return;
 299         count++;
 300
 301         rcu_read_lock();
 302         pr_notice("EDESTADDR occurred\n");
 303         pr_notice("CELL: %s err=%d\n", cell->name, cell->error);
 304         pr_notice("DNS: src=%u st=%u lc=%x\n",
 305                   cell->dns_source, cell->dns_status, cell->dns_lookup_count);
 306         pr_notice("VC: ut=%lx ix=%u ni=%hu fl=%hx err=%hd\n",
 307                   vc->untried, vc->index, vc->nr_iterations, vc->flags,
 308                   vc->cumul_error.error);
 309         pr_notice("VC: call  er=%d ac=%d r=%u\n",
 310                   vc->call_error, vc->call_abort_code, vc->call_responded);
 311
 312         if (vc->server_list) {
 313                 const struct afs_vlserver_list *sl = vc->server_list;
 314                 pr_notice("VC: SL nr=%u ix=%u\n",
 315                           sl->nr_servers, sl->index);
 316                 for (i = 0; i < sl->nr_servers; i++) {
 317                         const struct afs_vlserver *s = sl->servers[i].server;
 318                         pr_notice("VC: server %s+%hu fl=%lx E=%hd\n",
 319                                   s->name, s->port, s->flags, s->probe.error);
 320                         if (s->addresses) {
 321                                 const struct afs_addr_list *a =
 322                                         rcu_dereference(s->addresses);
 323                                 pr_notice("VC:  - nr=%u/%u/%u pf=%u\n",
 324                                           a->nr_ipv4, a->nr_addrs, a->max_addrs,
 325                                           a->preferred);
 326                                 pr_notice("VC:  - R=%lx F=%lx\n",
 327                                           a->responded, a->failed);
 328                                 if (a == vc->ac.alist)
 329                                         pr_notice("VC:  - current\n");
 330                         }
 331                 }
 332         }
 333
 334         pr_notice("AC: t=%lx ax=%u ni=%u\n",
 335                   vc->ac.tried, vc->ac.index, vc->ac.nr_iterations);
 336         rcu_read_unlock();
 337 }
 338
 339 /*
 340  * Tidy up a volume location server cursor and unlock the vnode.
 341  */
 342 int afs_end_vlserver_operation(struct afs_vl_cursor *vc)
 343 {
 344         struct afs_net *net = vc->cell->net;
 345
 346         switch (vc->cumul_error.error) {
 347         case -EDESTADDRREQ:
 348         case -EADDRNOTAVAIL:
 349         case -ENETUNREACH:
 350         case -EHOSTUNREACH:
 351                 afs_vl_dump_edestaddrreq(vc);
 352                 break;
 353         }
 354
 355         afs_end_cursor(&vc->ac);
 356         afs_put_vlserverlist(net, vc->server_list);
 357         return vc->cumul_error.error;
 358 }