mptcp/pm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Multipath TCP
   3  *
   4  * Copyright (c) 2019, Intel Corporation.
   5  */
   6 #define pr_fmt(fmt) "MPTCP: " fmt
   7
   8 #include <linux/kernel.h>
   9 #include <net/tcp.h>
  10 #include <net/mptcp.h>
  11 #include "protocol.h"
  12
  13 #include "mib.h"
  14
  15 /* path manager command handlers */
  16
  17 int mptcp_pm_announce_addr(struct mptcp_sock *msk,
  18                            const struct mptcp_addr_info *addr,
  19                            bool echo)
  20 {
  21         u8 add_addr = READ_ONCE(msk->pm.addr_signal);
  22
  23         pr_debug("msk=%p, local_id=%d, echo=%d", msk, addr->id, echo);
  24
  25         lockdep_assert_held(&msk->pm.lock);
  26
  27         if (add_addr &
  28             (echo ? BIT(MPTCP_ADD_ADDR_ECHO) : BIT(MPTCP_ADD_ADDR_SIGNAL))) {
  29                 pr_warn("addr_signal error, add_addr=%d, echo=%d", add_addr, echo);
  30                 return -EINVAL;
  31         }
  32
  33         if (echo) {
  34                 msk->pm.remote = *addr;
  35                 add_addr |= BIT(MPTCP_ADD_ADDR_ECHO);
  36         } else {
  37                 msk->pm.local = *addr;
  38                 add_addr |= BIT(MPTCP_ADD_ADDR_SIGNAL);
  39         }
  40         WRITE_ONCE(msk->pm.addr_signal, add_addr);
  41         return 0;
  42 }
  43
  44 int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list)
  45 {
  46         u8 rm_addr = READ_ONCE(msk->pm.addr_signal);
  47
  48         pr_debug("msk=%p, rm_list_nr=%d", msk, rm_list->nr);
  49
  50         if (rm_addr) {
  51                 pr_warn("addr_signal error, rm_addr=%d", rm_addr);
  52                 return -EINVAL;
  53         }
  54
  55         msk->pm.rm_list_tx = *rm_list;
  56         rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL);
  57         WRITE_ONCE(msk->pm.addr_signal, rm_addr);
  58         mptcp_pm_nl_addr_send_ack(msk);
  59         return 0;
  60 }
  61
  62 int mptcp_pm_remove_subflow(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list)
  63 {
  64         pr_debug("msk=%p, rm_list_nr=%d", msk, rm_list->nr);
  65
  66         spin_lock_bh(&msk->pm.lock);
  67         mptcp_pm_nl_rm_subflow_received(msk, rm_list);
  68         spin_unlock_bh(&msk->pm.lock);
  69         return 0;
  70 }
  71
  72 /* path manager event handlers */
  73
  74 void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side)
  75 {
  76         struct mptcp_pm_data *pm = &msk->pm;
  77
  78         pr_debug("msk=%p, token=%u side=%d", msk, msk->token, server_side);
  79
  80         WRITE_ONCE(pm->server_side, server_side);
  81         mptcp_event(MPTCP_EVENT_CREATED, msk, ssk, GFP_ATOMIC);
  82 }
  83
  84 bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk)
  85 {
  86         struct mptcp_pm_data *pm = &msk->pm;
  87         unsigned int subflows_max;
  88         int ret = 0;
  89
  90         if (mptcp_pm_is_userspace(msk)) {
  91                 if (mptcp_userspace_pm_active(msk)) {
  92                         spin_lock_bh(&pm->lock);
  93                         pm->subflows++;
  94                         spin_unlock_bh(&pm->lock);
  95                         return true;
  96                 }
  97                 return false;
  98         }
  99
 100         subflows_max = mptcp_pm_get_subflows_max(msk);
 101
 102         pr_debug("msk=%p subflows=%d max=%d allow=%d", msk, pm->subflows,
 103                  subflows_max, READ_ONCE(pm->accept_subflow));
 104
 105         /* try to avoid acquiring the lock below */
 106         if (!READ_ONCE(pm->accept_subflow))
 107                 return false;
 108
 109         spin_lock_bh(&pm->lock);
 110         if (READ_ONCE(pm->accept_subflow)) {
 111                 ret = pm->subflows < subflows_max;
 112                 if (ret && ++pm->subflows == subflows_max)
 113                         WRITE_ONCE(pm->accept_subflow, false);
 114         }
 115         spin_unlock_bh(&pm->lock);
 116
 117         return ret;
 118 }
 119
 120 /* return true if the new status bit is currently cleared, that is, this event
 121  * can be server, eventually by an already scheduled work
 122  */
 123 static bool mptcp_pm_schedule_work(struct mptcp_sock *msk,
 124                                    enum mptcp_pm_status new_status)
 125 {
 126         pr_debug("msk=%p status=%x new=%lx", msk, msk->pm.status,
 127                  BIT(new_status));
 128         if (msk->pm.status & BIT(new_status))
 129                 return false;
 130
 131         msk->pm.status |= BIT(new_status);
 132         mptcp_schedule_work((struct sock *)msk);
 133         return true;
 134 }
 135
 136 void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk, gfp_t gfp)
 137 {
 138         struct mptcp_pm_data *pm = &msk->pm;
 139         bool announce = false;
 140
 141         pr_debug("msk=%p", msk);
 142
 143         spin_lock_bh(&pm->lock);
 144
 145         /* mptcp_pm_fully_established() can be invoked by multiple
 146          * racing paths - accept() and check_fully_established()
 147          * be sure to serve this event only once.
 148          */
 149         if (READ_ONCE(pm->work_pending) &&
 150             !(msk->pm.status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)))
 151                 mptcp_pm_schedule_work(msk, MPTCP_PM_ESTABLISHED);
 152
 153         if ((msk->pm.status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)) == 0)
 154                 announce = true;
 155
 156         msk->pm.status |= BIT(MPTCP_PM_ALREADY_ESTABLISHED);
 157         spin_unlock_bh(&pm->lock);
 158
 159         if (announce)
 160                 mptcp_event(MPTCP_EVENT_ESTABLISHED, msk, ssk, gfp);
 161 }
 162
 163 void mptcp_pm_connection_closed(struct mptcp_sock *msk)
 164 {
 165         pr_debug("msk=%p", msk);
 166 }
 167
 168 void mptcp_pm_subflow_established(struct mptcp_sock *msk)
 169 {
 170         struct mptcp_pm_data *pm = &msk->pm;
 171
 172         pr_debug("msk=%p", msk);
 173
 174         if (!READ_ONCE(pm->work_pending))
 175                 return;
 176
 177         spin_lock_bh(&pm->lock);
 178
 179         if (READ_ONCE(pm->work_pending))
 180                 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);
 181
 182         spin_unlock_bh(&pm->lock);
 183 }
 184
 185 void mptcp_pm_subflow_check_next(struct mptcp_sock *msk, const struct sock *ssk,
 186                                  const struct mptcp_subflow_context *subflow)
 187 {
 188         struct mptcp_pm_data *pm = &msk->pm;
 189         bool update_subflows;
 190
 191         update_subflows = subflow->request_join || subflow->mp_join;
 192         if (mptcp_pm_is_userspace(msk)) {
 193                 if (update_subflows) {
 194                         spin_lock_bh(&pm->lock);
 195                         pm->subflows--;
 196                         spin_unlock_bh(&pm->lock);
 197                 }
 198                 return;
 199         }
 200
 201         if (!READ_ONCE(pm->work_pending) && !update_subflows)
 202                 return;
 203
 204         spin_lock_bh(&pm->lock);
 205         if (update_subflows)
 206                 __mptcp_pm_close_subflow(msk);
 207
 208         /* Even if this subflow is not really established, tell the PM to try
 209          * to pick the next ones, if possible.
 210          */
 211         if (mptcp_pm_nl_check_work_pending(msk))
 212                 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);
 213
 214         spin_unlock_bh(&pm->lock);
 215 }
 216
 217 void mptcp_pm_add_addr_received(const struct sock *ssk,
 218                                 const struct mptcp_addr_info *addr)
 219 {
 220         struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
 221         struct mptcp_sock *msk = mptcp_sk(subflow->conn);
 222         struct mptcp_pm_data *pm = &msk->pm;
 223
 224         pr_debug("msk=%p remote_id=%d accept=%d", msk, addr->id,
 225                  READ_ONCE(pm->accept_addr));
 226
 227         mptcp_event_addr_announced(ssk, addr);
 228
 229         spin_lock_bh(&pm->lock);
 230
 231         if (mptcp_pm_is_userspace(msk)) {
 232                 if (mptcp_userspace_pm_active(msk)) {
 233                         mptcp_pm_announce_addr(msk, addr, true);
 234                         mptcp_pm_add_addr_send_ack(msk);
 235                 } else {
 236                         __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
 237                 }
 238         } else if (!READ_ONCE(pm->accept_addr)) {
 239                 mptcp_pm_announce_addr(msk, addr, true);
 240                 mptcp_pm_add_addr_send_ack(msk);
 241         } else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) {
 242                 pm->remote = *addr;
 243         } else {
 244                 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
 245         }
 246
 247         spin_unlock_bh(&pm->lock);
 248 }
 249
 250 void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk,
 251                               const struct mptcp_addr_info *addr)
 252 {
 253         struct mptcp_pm_data *pm = &msk->pm;
 254
 255         pr_debug("msk=%p", msk);
 256
 257         spin_lock_bh(&pm->lock);
 258
 259         if (mptcp_lookup_anno_list_by_saddr(msk, addr) && READ_ONCE(pm->work_pending))
 260                 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);
 261
 262         spin_unlock_bh(&pm->lock);
 263 }
 264
 265 void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk)
 266 {
 267         if (!mptcp_pm_should_add_signal(msk))
 268                 return;
 269
 270         mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_SEND_ACK);
 271 }
 272
 273 void mptcp_pm_rm_addr_received(struct mptcp_sock *msk,
 274                                const struct mptcp_rm_list *rm_list)
 275 {
 276         struct mptcp_pm_data *pm = &msk->pm;
 277         u8 i;
 278
 279         pr_debug("msk=%p remote_ids_nr=%d", msk, rm_list->nr);
 280
 281         for (i = 0; i < rm_list->nr; i++)
 282                 mptcp_event_addr_removed(msk, rm_list->ids[i]);
 283
 284         spin_lock_bh(&pm->lock);
 285         if (mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED))
 286                 pm->rm_list_rx = *rm_list;
 287         else
 288                 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_RMADDRDROP);
 289         spin_unlock_bh(&pm->lock);
 290 }
 291
 292 void mptcp_pm_mp_prio_received(struct sock *ssk, u8 bkup)
 293 {
 294         struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
 295         struct sock *sk = subflow->conn;
 296         struct mptcp_sock *msk;
 297
 298         pr_debug("subflow->backup=%d, bkup=%d\n", subflow->backup, bkup);
 299         msk = mptcp_sk(sk);
 300         if (subflow->backup != bkup) {
 301                 subflow->backup = bkup;
 302                 mptcp_data_lock(sk);
 303                 if (!sock_owned_by_user(sk))
 304                         msk->last_snd = NULL;
 305                 else
 306                         __set_bit(MPTCP_RESET_SCHEDULER,  &msk->cb_flags);
 307                 mptcp_data_unlock(sk);
 308         }
 309
 310         mptcp_event(MPTCP_EVENT_SUB_PRIORITY, msk, ssk, GFP_ATOMIC);
 311 }
 312
 313 void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
 314 {
 315         struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 316         struct mptcp_sock *msk = mptcp_sk(subflow->conn);
 317
 318         pr_debug("fail_seq=%llu", fail_seq);
 319
 320         if (!READ_ONCE(msk->allow_infinite_fallback))
 321                 return;
 322
 323         if (!subflow->fail_tout) {
 324                 pr_debug("send MP_FAIL response and infinite map");
 325
 326                 subflow->send_mp_fail = 1;
 327                 subflow->send_infinite_map = 1;
 328                 tcp_send_ack(sk);
 329         } else {
 330                 pr_debug("MP_FAIL response received");
 331                 WRITE_ONCE(subflow->fail_tout, 0);
 332         }
 333 }
 334
 335 /* path manager helpers */
 336
 337 bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
 338                               unsigned int opt_size, unsigned int remaining,
 339                               struct mptcp_addr_info *addr, bool *echo,
 340                               bool *drop_other_suboptions)
 341 {
 342         int ret = false;
 343         u8 add_addr;
 344         u8 family;
 345         bool port;
 346
 347         spin_lock_bh(&msk->pm.lock);
 348
 349         /* double check after the lock is acquired */
 350         if (!mptcp_pm_should_add_signal(msk))
 351                 goto out_unlock;
 352
 353         /* always drop every other options for pure ack ADD_ADDR; this is a
 354          * plain dup-ack from TCP perspective. The other MPTCP-relevant info,
 355          * if any, will be carried by the 'original' TCP ack
 356          */
 357         if (skb && skb_is_tcp_pure_ack(skb)) {
 358                 remaining += opt_size;
 359                 *drop_other_suboptions = true;
 360         }
 361
 362         *echo = mptcp_pm_should_add_signal_echo(msk);
 363         port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port);
 364
 365         family = *echo ? msk->pm.remote.family : msk->pm.local.family;
 366         if (remaining < mptcp_add_addr_len(family, *echo, port))
 367                 goto out_unlock;
 368
 369         if (*echo) {
 370                 *addr = msk->pm.remote;
 371                 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO);
 372         } else {
 373                 *addr = msk->pm.local;
 374                 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL);
 375         }
 376         WRITE_ONCE(msk->pm.addr_signal, add_addr);
 377         ret = true;
 378
 379 out_unlock:
 380         spin_unlock_bh(&msk->pm.lock);
 381         return ret;
 382 }
 383
 384 bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
 385                              struct mptcp_rm_list *rm_list)
 386 {
 387         int ret = false, len;
 388         u8 rm_addr;
 389
 390         spin_lock_bh(&msk->pm.lock);
 391
 392         /* double check after the lock is acquired */
 393         if (!mptcp_pm_should_rm_signal(msk))
 394                 goto out_unlock;
 395
 396         rm_addr = msk->pm.addr_signal & ~BIT(MPTCP_RM_ADDR_SIGNAL);
 397         len = mptcp_rm_addr_len(&msk->pm.rm_list_tx);
 398         if (len < 0) {
 399                 WRITE_ONCE(msk->pm.addr_signal, rm_addr);
 400                 goto out_unlock;
 401         }
 402         if (remaining < len)
 403                 goto out_unlock;
 404
 405         *rm_list = msk->pm.rm_list_tx;
 406         WRITE_ONCE(msk->pm.addr_signal, rm_addr);
 407         ret = true;
 408
 409 out_unlock:
 410         spin_unlock_bh(&msk->pm.lock);
 411         return ret;
 412 }
 413
 414 int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc)
 415 {
 416         return mptcp_pm_nl_get_local_id(msk, skc);
 417 }
 418
 419 void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk)
 420 {
 421         struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
 422         u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp);
 423
 424         /* keep track of rtx periods with no progress */
 425         if (!subflow->stale_count) {
 426                 subflow->stale_rcv_tstamp = rcv_tstamp;
 427                 subflow->stale_count++;
 428         } else if (subflow->stale_rcv_tstamp == rcv_tstamp) {
 429                 if (subflow->stale_count < U8_MAX)
 430                         subflow->stale_count++;
 431                 mptcp_pm_nl_subflow_chk_stale(msk, ssk);
 432         } else {
 433                 subflow->stale_count = 0;
 434                 mptcp_subflow_set_active(subflow);
 435         }
 436 }
 437
 438 /* if sk is ipv4 or ipv6_only allows only same-family local and remote addresses,
 439  * otherwise allow any matching local/remote pair
 440  */
 441 bool mptcp_pm_addr_families_match(const struct sock *sk,
 442                                   const struct mptcp_addr_info *loc,
 443                                   const struct mptcp_addr_info *rem)
 444 {
 445         bool mptcp_is_v4 = sk->sk_family == AF_INET;
 446
 447 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
 448         bool loc_is_v4 = loc->family == AF_INET || ipv6_addr_v4mapped(&loc->addr6);
 449         bool rem_is_v4 = rem->family == AF_INET || ipv6_addr_v4mapped(&rem->addr6);
 450
 451         if (mptcp_is_v4)
 452                 return loc_is_v4 && rem_is_v4;
 453
 454         if (ipv6_only_sock(sk))
 455                 return !loc_is_v4 && !rem_is_v4;
 456
 457         return loc_is_v4 == rem_is_v4;
 458 #else
 459         return mptcp_is_v4 && loc->family == AF_INET && rem->family == AF_INET;
 460 #endif
 461 }
 462
 463 void mptcp_pm_data_reset(struct mptcp_sock *msk)
 464 {
 465         u8 pm_type = mptcp_get_pm_type(sock_net((struct sock *)msk));
 466         struct mptcp_pm_data *pm = &msk->pm;
 467
 468         pm->add_addr_signaled = 0;
 469         pm->add_addr_accepted = 0;
 470         pm->local_addr_used = 0;
 471         pm->subflows = 0;
 472         pm->rm_list_tx.nr = 0;
 473         pm->rm_list_rx.nr = 0;
 474         WRITE_ONCE(pm->pm_type, pm_type);
 475
 476         if (pm_type == MPTCP_PM_TYPE_KERNEL) {
 477                 bool subflows_allowed = !!mptcp_pm_get_subflows_max(msk);
 478
 479                 /* pm->work_pending must be only be set to 'true' when
 480                  * pm->pm_type is set to MPTCP_PM_TYPE_KERNEL
 481                  */
 482                 WRITE_ONCE(pm->work_pending,
 483                            (!!mptcp_pm_get_local_addr_max(msk) &&
 484                             subflows_allowed) ||
 485                            !!mptcp_pm_get_add_addr_signal_max(msk));
 486                 WRITE_ONCE(pm->accept_addr,
 487                            !!mptcp_pm_get_add_addr_accept_max(msk) &&
 488                            subflows_allowed);
 489                 WRITE_ONCE(pm->accept_subflow, subflows_allowed);
 490         } else {
 491                 WRITE_ONCE(pm->work_pending, 0);
 492                 WRITE_ONCE(pm->accept_addr, 0);
 493                 WRITE_ONCE(pm->accept_subflow, 0);
 494         }
 495
 496         WRITE_ONCE(pm->addr_signal, 0);
 497         WRITE_ONCE(pm->remote_deny_join_id0, false);
 498         pm->status = 0;
 499         bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
 500 }
 501
 502 void mptcp_pm_data_init(struct mptcp_sock *msk)
 503 {
 504         spin_lock_init(&msk->pm.lock);
 505         INIT_LIST_HEAD(&msk->pm.anno_list);
 506         INIT_LIST_HEAD(&msk->pm.userspace_pm_local_addr_list);
 507         mptcp_pm_data_reset(msk);
 508 }
 509
 510 void __init mptcp_pm_init(void)
 511 {
 512         mptcp_pm_nl_init();
 513 }