ipv4/tcp_ao.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * INET         An implementation of the TCP Authentication Option (TCP-AO).
   4  *              See RFC5925.
   5  *
   6  * Authors:     Dmitry Safonov <dima@arista.com>
   7  *              Francesco Ruggeri <fruggeri@arista.com>
   8  *              Salam Noureddine <noureddine@arista.com>
   9  */
  10 #define pr_fmt(fmt) "TCP: " fmt
  11
  12 #include <crypto/hash.h>
  13 #include <linux/inetdevice.h>
  14 #include <linux/tcp.h>
  15
  16 #include <net/tcp.h>
  17 #include <net/ipv6.h>
  18 #include <net/icmp.h>
  19
  20 DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_ao_needed, HZ);
  21
  22 int tcp_ao_calc_traffic_key(struct tcp_ao_key *mkt, u8 *key, void *ctx,
  23                             unsigned int len, struct tcp_sigpool *hp)
  24 {
  25         struct scatterlist sg;
  26         int ret;
  27
  28         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp->req),
  29                                 mkt->key, mkt->keylen))
  30                 goto clear_hash;
  31
  32         ret = crypto_ahash_init(hp->req);
  33         if (ret)
  34                 goto clear_hash;
  35
  36         sg_init_one(&sg, ctx, len);
  37         ahash_request_set_crypt(hp->req, &sg, key, len);
  38         crypto_ahash_update(hp->req);
  39
  40         ret = crypto_ahash_final(hp->req);
  41         if (ret)
  42                 goto clear_hash;
  43
  44         return 0;
  45 clear_hash:
  46         memset(key, 0, tcp_ao_digest_size(mkt));
  47         return 1;
  48 }
  49
  50 bool tcp_ao_ignore_icmp(const struct sock *sk, int family, int type, int code)
  51 {
  52         bool ignore_icmp = false;
  53         struct tcp_ao_info *ao;
  54
  55         if (!static_branch_unlikely(&tcp_ao_needed.key))
  56                 return false;
  57
  58         /* RFC5925, 7.8:
  59          * >> A TCP-AO implementation MUST default to ignore incoming ICMPv4
  60          * messages of Type 3 (destination unreachable), Codes 2-4 (protocol
  61          * unreachable, port unreachable, and fragmentation needed -- ’hard
  62          * errors’), and ICMPv6 Type 1 (destination unreachable), Code 1
  63          * (administratively prohibited) and Code 4 (port unreachable) intended
  64          * for connections in synchronized states (ESTABLISHED, FIN-WAIT-1, FIN-
  65          * WAIT-2, CLOSE-WAIT, CLOSING, LAST-ACK, TIME-WAIT) that match MKTs.
  66          */
  67         if (family == AF_INET) {
  68                 if (type != ICMP_DEST_UNREACH)
  69                         return false;
  70                 if (code < ICMP_PROT_UNREACH || code > ICMP_FRAG_NEEDED)
  71                         return false;
  72         } else {
  73                 if (type != ICMPV6_DEST_UNREACH)
  74                         return false;
  75                 if (code != ICMPV6_ADM_PROHIBITED && code != ICMPV6_PORT_UNREACH)
  76                         return false;
  77         }
  78
  79         rcu_read_lock();
  80         switch (sk->sk_state) {
  81         case TCP_TIME_WAIT:
  82                 ao = rcu_dereference(tcp_twsk(sk)->ao_info);
  83                 break;
  84         case TCP_SYN_SENT:
  85         case TCP_SYN_RECV:
  86         case TCP_LISTEN:
  87         case TCP_NEW_SYN_RECV:
  88                 /* RFC5925 specifies to ignore ICMPs *only* on connections
  89                  * in synchronized states.
  90                  */
  91                 rcu_read_unlock();
  92                 return false;
  93         default:
  94                 ao = rcu_dereference(tcp_sk(sk)->ao_info);
  95         }
  96
  97         if (ao && !ao->accept_icmps) {
  98                 ignore_icmp = true;
  99                 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAODROPPEDICMPS);
 100                 atomic64_inc(&ao->counters.dropped_icmp);
 101         }
 102         rcu_read_unlock();
 103
 104         return ignore_icmp;
 105 }
 106
 107 /* Optimized version of tcp_ao_do_lookup(): only for sockets for which
 108  * it's known that the keys in ao_info are matching peer's
 109  * family/address/VRF/etc.
 110  */
 111 struct tcp_ao_key *tcp_ao_established_key(struct tcp_ao_info *ao,
 112                                           int sndid, int rcvid)
 113 {
 114         struct tcp_ao_key *key;
 115
 116         hlist_for_each_entry_rcu(key, &ao->head, node) {
 117                 if ((sndid >= 0 && key->sndid != sndid) ||
 118                     (rcvid >= 0 && key->rcvid != rcvid))
 119                         continue;
 120                 return key;
 121         }
 122
 123         return NULL;
 124 }
 125
 126 static int ipv4_prefix_cmp(const struct in_addr *addr1,
 127                            const struct in_addr *addr2,
 128                            unsigned int prefixlen)
 129 {
 130         __be32 mask = inet_make_mask(prefixlen);
 131         __be32 a1 = addr1->s_addr & mask;
 132         __be32 a2 = addr2->s_addr & mask;
 133
 134         if (a1 == a2)
 135                 return 0;
 136         return memcmp(&a1, &a2, sizeof(a1));
 137 }
 138
 139 static int __tcp_ao_key_cmp(const struct tcp_ao_key *key, int l3index,
 140                             const union tcp_ao_addr *addr, u8 prefixlen,
 141                             int family, int sndid, int rcvid)
 142 {
 143         if (sndid >= 0 && key->sndid != sndid)
 144                 return (key->sndid > sndid) ? 1 : -1;
 145         if (rcvid >= 0 && key->rcvid != rcvid)
 146                 return (key->rcvid > rcvid) ? 1 : -1;
 147         if (l3index >= 0 && (key->keyflags & TCP_AO_KEYF_IFINDEX)) {
 148                 if (key->l3index != l3index)
 149                         return (key->l3index > l3index) ? 1 : -1;
 150         }
 151
 152         if (family == AF_UNSPEC)
 153                 return 0;
 154         if (key->family != family)
 155                 return (key->family > family) ? 1 : -1;
 156
 157         if (family == AF_INET) {
 158                 if (ntohl(key->addr.a4.s_addr) == INADDR_ANY)
 159                         return 0;
 160                 if (ntohl(addr->a4.s_addr) == INADDR_ANY)
 161                         return 0;
 162                 return ipv4_prefix_cmp(&key->addr.a4, &addr->a4, prefixlen);
 163 #if IS_ENABLED(CONFIG_IPV6)
 164         } else {
 165                 if (ipv6_addr_any(&key->addr.a6) || ipv6_addr_any(&addr->a6))
 166                         return 0;
 167                 if (ipv6_prefix_equal(&key->addr.a6, &addr->a6, prefixlen))
 168                         return 0;
 169                 return memcmp(&key->addr.a6, &addr->a6, sizeof(addr->a6));
 170 #endif
 171         }
 172         return -1;
 173 }
 174
 175 static int tcp_ao_key_cmp(const struct tcp_ao_key *key, int l3index,
 176                           const union tcp_ao_addr *addr, u8 prefixlen,
 177                           int family, int sndid, int rcvid)
 178 {
 179 #if IS_ENABLED(CONFIG_IPV6)
 180         if (family == AF_INET6 && ipv6_addr_v4mapped(&addr->a6)) {
 181                 __be32 addr4 = addr->a6.s6_addr32[3];
 182
 183                 return __tcp_ao_key_cmp(key, l3index,
 184                                         (union tcp_ao_addr *)&addr4,
 185                                         prefixlen, AF_INET, sndid, rcvid);
 186         }
 187 #endif
 188         return __tcp_ao_key_cmp(key, l3index, addr,
 189                                 prefixlen, family, sndid, rcvid);
 190 }
 191
 192 static struct tcp_ao_key *__tcp_ao_do_lookup(const struct sock *sk, int l3index,
 193                 const union tcp_ao_addr *addr, int family, u8 prefix,
 194                 int sndid, int rcvid)
 195 {
 196         struct tcp_ao_key *key;
 197         struct tcp_ao_info *ao;
 198
 199         if (!static_branch_unlikely(&tcp_ao_needed.key))
 200                 return NULL;
 201
 202         ao = rcu_dereference_check(tcp_sk(sk)->ao_info,
 203                                    lockdep_sock_is_held(sk));
 204         if (!ao)
 205                 return NULL;
 206
 207         hlist_for_each_entry_rcu(key, &ao->head, node) {
 208                 u8 prefixlen = min(prefix, key->prefixlen);
 209
 210                 if (!tcp_ao_key_cmp(key, l3index, addr, prefixlen,
 211                                     family, sndid, rcvid))
 212                         return key;
 213         }
 214         return NULL;
 215 }
 216
 217 struct tcp_ao_key *tcp_ao_do_lookup(const struct sock *sk, int l3index,
 218                                     const union tcp_ao_addr *addr,
 219                                     int family, int sndid, int rcvid)
 220 {
 221         return __tcp_ao_do_lookup(sk, l3index, addr, family, U8_MAX, sndid, rcvid);
 222 }
 223
 224 static struct tcp_ao_info *tcp_ao_alloc_info(gfp_t flags)
 225 {
 226         struct tcp_ao_info *ao;
 227
 228         ao = kzalloc(sizeof(*ao), flags);
 229         if (!ao)
 230                 return NULL;
 231         INIT_HLIST_HEAD(&ao->head);
 232         refcount_set(&ao->refcnt, 1);
 233
 234         return ao;
 235 }
 236
 237 static void tcp_ao_link_mkt(struct tcp_ao_info *ao, struct tcp_ao_key *mkt)
 238 {
 239         hlist_add_head_rcu(&mkt->node, &ao->head);
 240 }
 241
 242 static struct tcp_ao_key *tcp_ao_copy_key(struct sock *sk,
 243                                           struct tcp_ao_key *key)
 244 {
 245         struct tcp_ao_key *new_key;
 246
 247         new_key = sock_kmalloc(sk, tcp_ao_sizeof_key(key),
 248                                GFP_ATOMIC);
 249         if (!new_key)
 250                 return NULL;
 251
 252         *new_key = *key;
 253         INIT_HLIST_NODE(&new_key->node);
 254         tcp_sigpool_get(new_key->tcp_sigpool_id);
 255         atomic64_set(&new_key->pkt_good, 0);
 256         atomic64_set(&new_key->pkt_bad, 0);
 257
 258         return new_key;
 259 }
 260
 261 static void tcp_ao_key_free_rcu(struct rcu_head *head)
 262 {
 263         struct tcp_ao_key *key = container_of(head, struct tcp_ao_key, rcu);
 264
 265         tcp_sigpool_release(key->tcp_sigpool_id);
 266         kfree_sensitive(key);
 267 }
 268
 269 void tcp_ao_destroy_sock(struct sock *sk, bool twsk)
 270 {
 271         struct tcp_ao_info *ao;
 272         struct tcp_ao_key *key;
 273         struct hlist_node *n;
 274
 275         if (twsk) {
 276                 ao = rcu_dereference_protected(tcp_twsk(sk)->ao_info, 1);
 277                 tcp_twsk(sk)->ao_info = NULL;
 278         } else {
 279                 ao = rcu_dereference_protected(tcp_sk(sk)->ao_info, 1);
 280                 tcp_sk(sk)->ao_info = NULL;
 281         }
 282
 283         if (!ao || !refcount_dec_and_test(&ao->refcnt))
 284                 return;
 285
 286         hlist_for_each_entry_safe(key, n, &ao->head, node) {
 287                 hlist_del_rcu(&key->node);
 288                 if (!twsk)
 289                         atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
 290                 call_rcu(&key->rcu, tcp_ao_key_free_rcu);
 291         }
 292
 293         kfree_rcu(ao, rcu);
 294         static_branch_slow_dec_deferred(&tcp_ao_needed);
 295 }
 296
 297 void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp)
 298 {
 299         struct tcp_ao_info *ao_info = rcu_dereference_protected(tp->ao_info, 1);
 300
 301         if (ao_info) {
 302                 struct tcp_ao_key *key;
 303                 struct hlist_node *n;
 304                 int omem = 0;
 305
 306                 hlist_for_each_entry_safe(key, n, &ao_info->head, node) {
 307                         omem += tcp_ao_sizeof_key(key);
 308                 }
 309
 310                 refcount_inc(&ao_info->refcnt);
 311                 atomic_sub(omem, &(((struct sock *)tp)->sk_omem_alloc));
 312                 rcu_assign_pointer(tcptw->ao_info, ao_info);
 313         } else {
 314                 tcptw->ao_info = NULL;
 315         }
 316 }
 317
 318 /* 4 tuple and ISNs are expected in NBO */
 319 static int tcp_v4_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
 320                               __be32 saddr, __be32 daddr,
 321                               __be16 sport, __be16 dport,
 322                               __be32 sisn,  __be32 disn)
 323 {
 324         /* See RFC5926 3.1.1 */
 325         struct kdf_input_block {
 326                 u8                      counter;
 327                 u8                      label[6];
 328                 struct tcp4_ao_context  ctx;
 329                 __be16                  outlen;
 330         } __packed * tmp;
 331         struct tcp_sigpool hp;
 332         int err;
 333
 334         err = tcp_sigpool_start(mkt->tcp_sigpool_id, &hp);
 335         if (err)
 336                 return err;
 337
 338         tmp = hp.scratch;
 339         tmp->counter    = 1;
 340         memcpy(tmp->label, "TCP-AO", 6);
 341         tmp->ctx.saddr  = saddr;
 342         tmp->ctx.daddr  = daddr;
 343         tmp->ctx.sport  = sport;
 344         tmp->ctx.dport  = dport;
 345         tmp->ctx.sisn   = sisn;
 346         tmp->ctx.disn   = disn;
 347         tmp->outlen     = htons(tcp_ao_digest_size(mkt) * 8); /* in bits */
 348
 349         err = tcp_ao_calc_traffic_key(mkt, key, tmp, sizeof(*tmp), &hp);
 350         tcp_sigpool_end(&hp);
 351
 352         return err;
 353 }
 354
 355 int tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
 356                           const struct sock *sk,
 357                           __be32 sisn, __be32 disn, bool send)
 358 {
 359         if (send)
 360                 return tcp_v4_ao_calc_key(mkt, key, sk->sk_rcv_saddr,
 361                                           sk->sk_daddr, htons(sk->sk_num),
 362                                           sk->sk_dport, sisn, disn);
 363         else
 364                 return tcp_v4_ao_calc_key(mkt, key, sk->sk_daddr,
 365                                           sk->sk_rcv_saddr, sk->sk_dport,
 366                                           htons(sk->sk_num), disn, sisn);
 367 }
 368
 369 static int tcp_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
 370                               const struct sock *sk,
 371                               __be32 sisn, __be32 disn, bool send)
 372 {
 373         if (mkt->family == AF_INET)
 374                 return tcp_v4_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
 375 #if IS_ENABLED(CONFIG_IPV6)
 376         else if (mkt->family == AF_INET6)
 377                 return tcp_v6_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
 378 #endif
 379         else
 380                 return -EOPNOTSUPP;
 381 }
 382
 383 int tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
 384                            struct request_sock *req)
 385 {
 386         struct inet_request_sock *ireq = inet_rsk(req);
 387
 388         return tcp_v4_ao_calc_key(mkt, key,
 389                                   ireq->ir_loc_addr, ireq->ir_rmt_addr,
 390                                   htons(ireq->ir_num), ireq->ir_rmt_port,
 391                                   htonl(tcp_rsk(req)->snt_isn),
 392                                   htonl(tcp_rsk(req)->rcv_isn));
 393 }
 394
 395 static int tcp_v4_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
 396                                   const struct sk_buff *skb,
 397                                   __be32 sisn, __be32 disn)
 398 {
 399         const struct iphdr *iph = ip_hdr(skb);
 400         const struct tcphdr *th = tcp_hdr(skb);
 401
 402         return tcp_v4_ao_calc_key(mkt, key, iph->saddr, iph->daddr,
 403                                   th->source, th->dest, sisn, disn);
 404 }
 405
 406 static int tcp_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
 407                                const struct sk_buff *skb,
 408                                __be32 sisn, __be32 disn, int family)
 409 {
 410         if (family == AF_INET)
 411                 return tcp_v4_ao_calc_key_skb(mkt, key, skb, sisn, disn);
 412 #if IS_ENABLED(CONFIG_IPV6)
 413         else if (family == AF_INET6)
 414                 return tcp_v6_ao_calc_key_skb(mkt, key, skb, sisn, disn);
 415 #endif
 416         return -EAFNOSUPPORT;
 417 }
 418
 419 static int tcp_v4_ao_hash_pseudoheader(struct tcp_sigpool *hp,
 420                                        __be32 daddr, __be32 saddr,
 421                                        int nbytes)
 422 {
 423         struct tcp4_pseudohdr *bp;
 424         struct scatterlist sg;
 425
 426         bp = hp->scratch;
 427         bp->saddr = saddr;
 428         bp->daddr = daddr;
 429         bp->pad = 0;
 430         bp->protocol = IPPROTO_TCP;
 431         bp->len = cpu_to_be16(nbytes);
 432
 433         sg_init_one(&sg, bp, sizeof(*bp));
 434         ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
 435         return crypto_ahash_update(hp->req);
 436 }
 437
 438 static int tcp_ao_hash_pseudoheader(unsigned short int family,
 439                                     const struct sock *sk,
 440                                     const struct sk_buff *skb,
 441                                     struct tcp_sigpool *hp, int nbytes)
 442 {
 443         const struct tcphdr *th = tcp_hdr(skb);
 444
 445         /* TODO: Can we rely on checksum being zero to mean outbound pkt? */
 446         if (!th->check) {
 447                 if (family == AF_INET)
 448                         return tcp_v4_ao_hash_pseudoheader(hp, sk->sk_daddr,
 449                                         sk->sk_rcv_saddr, skb->len);
 450 #if IS_ENABLED(CONFIG_IPV6)
 451                 else if (family == AF_INET6)
 452                         return tcp_v6_ao_hash_pseudoheader(hp, &sk->sk_v6_daddr,
 453                                         &sk->sk_v6_rcv_saddr, skb->len);
 454 #endif
 455                 else
 456                         return -EAFNOSUPPORT;
 457         }
 458
 459         if (family == AF_INET) {
 460                 const struct iphdr *iph = ip_hdr(skb);
 461
 462                 return tcp_v4_ao_hash_pseudoheader(hp, iph->daddr,
 463                                 iph->saddr, skb->len);
 464 #if IS_ENABLED(CONFIG_IPV6)
 465         } else if (family == AF_INET6) {
 466                 const struct ipv6hdr *iph = ipv6_hdr(skb);
 467
 468                 return tcp_v6_ao_hash_pseudoheader(hp, &iph->daddr,
 469                                 &iph->saddr, skb->len);
 470 #endif
 471         }
 472         return -EAFNOSUPPORT;
 473 }
 474
 475 u32 tcp_ao_compute_sne(u32 next_sne, u32 next_seq, u32 seq)
 476 {
 477         u32 sne = next_sne;
 478
 479         if (before(seq, next_seq)) {
 480                 if (seq > next_seq)
 481                         sne--;
 482         } else {
 483                 if (seq < next_seq)
 484                         sne++;
 485         }
 486
 487         return sne;
 488 }
 489
 490 /* tcp_ao_hash_sne(struct tcp_sigpool *hp)
 491  * @hp  - used for hashing
 492  * @sne - sne value
 493  */
 494 static int tcp_ao_hash_sne(struct tcp_sigpool *hp, u32 sne)
 495 {
 496         struct scatterlist sg;
 497         __be32 *bp;
 498
 499         bp = (__be32 *)hp->scratch;
 500         *bp = htonl(sne);
 501
 502         sg_init_one(&sg, bp, sizeof(*bp));
 503         ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
 504         return crypto_ahash_update(hp->req);
 505 }
 506
 507 static int tcp_ao_hash_header(struct tcp_sigpool *hp,
 508                               const struct tcphdr *th,
 509                               bool exclude_options, u8 *hash,
 510                               int hash_offset, int hash_len)
 511 {
 512         int err, len = th->doff << 2;
 513         struct scatterlist sg;
 514         u8 *hdr = hp->scratch;
 515
 516         /* We are not allowed to change tcphdr, make a local copy */
 517         if (exclude_options) {
 518                 len = sizeof(*th) + sizeof(struct tcp_ao_hdr) + hash_len;
 519                 memcpy(hdr, th, sizeof(*th));
 520                 memcpy(hdr + sizeof(*th),
 521                        (u8 *)th + hash_offset - sizeof(struct tcp_ao_hdr),
 522                        sizeof(struct tcp_ao_hdr));
 523                 memset(hdr + sizeof(*th) + sizeof(struct tcp_ao_hdr),
 524                        0, hash_len);
 525                 ((struct tcphdr *)hdr)->check = 0;
 526         } else {
 527                 len = th->doff << 2;
 528                 memcpy(hdr, th, len);
 529                 /* zero out tcp-ao hash */
 530                 ((struct tcphdr *)hdr)->check = 0;
 531                 memset(hdr + hash_offset, 0, hash_len);
 532         }
 533
 534         sg_init_one(&sg, hdr, len);
 535         ahash_request_set_crypt(hp->req, &sg, NULL, len);
 536         err = crypto_ahash_update(hp->req);
 537         WARN_ON_ONCE(err != 0);
 538         return err;
 539 }
 540
 541 int tcp_ao_hash_hdr(unsigned short int family, char *ao_hash,
 542                     struct tcp_ao_key *key, const u8 *tkey,
 543                     const union tcp_ao_addr *daddr,
 544                     const union tcp_ao_addr *saddr,
 545                     const struct tcphdr *th, u32 sne)
 546 {
 547         int tkey_len = tcp_ao_digest_size(key);
 548         int hash_offset = ao_hash - (char *)th;
 549         struct tcp_sigpool hp;
 550         void *hash_buf = NULL;
 551
 552         hash_buf = kmalloc(tkey_len, GFP_ATOMIC);
 553         if (!hash_buf)
 554                 goto clear_hash_noput;
 555
 556         if (tcp_sigpool_start(key->tcp_sigpool_id, &hp))
 557                 goto clear_hash_noput;
 558
 559         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp.req), tkey, tkey_len))
 560                 goto clear_hash;
 561
 562         if (crypto_ahash_init(hp.req))
 563                 goto clear_hash;
 564
 565         if (tcp_ao_hash_sne(&hp, sne))
 566                 goto clear_hash;
 567         if (family == AF_INET) {
 568                 if (tcp_v4_ao_hash_pseudoheader(&hp, daddr->a4.s_addr,
 569                                                 saddr->a4.s_addr, th->doff * 4))
 570                         goto clear_hash;
 571 #if IS_ENABLED(CONFIG_IPV6)
 572         } else if (family == AF_INET6) {
 573                 if (tcp_v6_ao_hash_pseudoheader(&hp, &daddr->a6,
 574                                                 &saddr->a6, th->doff * 4))
 575                         goto clear_hash;
 576 #endif
 577         } else {
 578                 WARN_ON_ONCE(1);
 579                 goto clear_hash;
 580         }
 581         if (tcp_ao_hash_header(&hp, th,
 582                                !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
 583                                ao_hash, hash_offset, tcp_ao_maclen(key)))
 584                 goto clear_hash;
 585         ahash_request_set_crypt(hp.req, NULL, hash_buf, 0);
 586         if (crypto_ahash_final(hp.req))
 587                 goto clear_hash;
 588
 589         memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
 590         tcp_sigpool_end(&hp);
 591         kfree(hash_buf);
 592         return 0;
 593
 594 clear_hash:
 595         tcp_sigpool_end(&hp);
 596 clear_hash_noput:
 597         memset(ao_hash, 0, tcp_ao_maclen(key));
 598         kfree(hash_buf);
 599         return 1;
 600 }
 601
 602 int tcp_ao_hash_skb(unsigned short int family,
 603                     char *ao_hash, struct tcp_ao_key *key,
 604                     const struct sock *sk, const struct sk_buff *skb,
 605                     const u8 *tkey, int hash_offset, u32 sne)
 606 {
 607         const struct tcphdr *th = tcp_hdr(skb);
 608         int tkey_len = tcp_ao_digest_size(key);
 609         struct tcp_sigpool hp;
 610         void *hash_buf = NULL;
 611
 612         hash_buf = kmalloc(tkey_len, GFP_ATOMIC);
 613         if (!hash_buf)
 614                 goto clear_hash_noput;
 615
 616         if (tcp_sigpool_start(key->tcp_sigpool_id, &hp))
 617                 goto clear_hash_noput;
 618
 619         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp.req), tkey, tkey_len))
 620                 goto clear_hash;
 621
 622         /* For now use sha1 by default. Depends on alg in tcp_ao_key */
 623         if (crypto_ahash_init(hp.req))
 624                 goto clear_hash;
 625
 626         if (tcp_ao_hash_sne(&hp, sne))
 627                 goto clear_hash;
 628         if (tcp_ao_hash_pseudoheader(family, sk, skb, &hp, skb->len))
 629                 goto clear_hash;
 630         if (tcp_ao_hash_header(&hp, th,
 631                                !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
 632                                ao_hash, hash_offset, tcp_ao_maclen(key)))
 633                 goto clear_hash;
 634         if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
 635                 goto clear_hash;
 636         ahash_request_set_crypt(hp.req, NULL, hash_buf, 0);
 637         if (crypto_ahash_final(hp.req))
 638                 goto clear_hash;
 639
 640         memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
 641         tcp_sigpool_end(&hp);
 642         kfree(hash_buf);
 643         return 0;
 644
 645 clear_hash:
 646         tcp_sigpool_end(&hp);
 647 clear_hash_noput:
 648         memset(ao_hash, 0, tcp_ao_maclen(key));
 649         kfree(hash_buf);
 650         return 1;
 651 }
 652
 653 int tcp_v4_ao_hash_skb(char *ao_hash, struct tcp_ao_key *key,
 654                        const struct sock *sk, const struct sk_buff *skb,
 655                        const u8 *tkey, int hash_offset, u32 sne)
 656 {
 657         return tcp_ao_hash_skb(AF_INET, ao_hash, key, sk, skb,
 658                                tkey, hash_offset, sne);
 659 }
 660
 661 int tcp_v4_ao_synack_hash(char *ao_hash, struct tcp_ao_key *ao_key,
 662                           struct request_sock *req, const struct sk_buff *skb,
 663                           int hash_offset, u32 sne)
 664 {
 665         void *hash_buf = NULL;
 666         int err;
 667
 668         hash_buf = kmalloc(tcp_ao_digest_size(ao_key), GFP_ATOMIC);
 669         if (!hash_buf)
 670                 return -ENOMEM;
 671
 672         err = tcp_v4_ao_calc_key_rsk(ao_key, hash_buf, req);
 673         if (err)
 674                 goto out;
 675
 676         err = tcp_ao_hash_skb(AF_INET, ao_hash, ao_key, req_to_sk(req), skb,
 677                               hash_buf, hash_offset, sne);
 678 out:
 679         kfree(hash_buf);
 680         return err;
 681 }
 682
 683 struct tcp_ao_key *tcp_v4_ao_lookup_rsk(const struct sock *sk,
 684                                         struct request_sock *req,
 685                                         int sndid, int rcvid)
 686 {
 687         struct inet_request_sock *ireq = inet_rsk(req);
 688         union tcp_ao_addr *addr = (union tcp_ao_addr *)&ireq->ir_rmt_addr;
 689         int l3index;
 690
 691         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
 692         return tcp_ao_do_lookup(sk, l3index, addr, AF_INET, sndid, rcvid);
 693 }
 694
 695 struct tcp_ao_key *tcp_v4_ao_lookup(const struct sock *sk, struct sock *addr_sk,
 696                                     int sndid, int rcvid)
 697 {
 698         int l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 699                                                      addr_sk->sk_bound_dev_if);
 700         union tcp_ao_addr *addr = (union tcp_ao_addr *)&addr_sk->sk_daddr;
 701
 702         return tcp_ao_do_lookup(sk, l3index, addr, AF_INET, sndid, rcvid);
 703 }
 704
 705 int tcp_ao_prepare_reset(const struct sock *sk, struct sk_buff *skb,
 706                          const struct tcp_ao_hdr *aoh, int l3index, u32 seq,
 707                          struct tcp_ao_key **key, char **traffic_key,
 708                          bool *allocated_traffic_key, u8 *keyid, u32 *sne)
 709 {
 710         const struct tcphdr *th = tcp_hdr(skb);
 711         struct tcp_ao_info *ao_info;
 712
 713         *allocated_traffic_key = false;
 714         /* If there's no socket - than initial sisn/disn are unknown.
 715          * Drop the segment. RFC5925 (7.7) advises to require graceful
 716          * restart [RFC4724]. Alternatively, the RFC5925 advises to
 717          * save/restore traffic keys before/after reboot.
 718          * Linux TCP-AO support provides TCP_AO_ADD_KEY and TCP_AO_REPAIR
 719          * options to restore a socket post-reboot.
 720          */
 721         if (!sk)
 722                 return -ENOTCONN;
 723
 724         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV)) {
 725                 unsigned int family = READ_ONCE(sk->sk_family);
 726                 union tcp_ao_addr *addr;
 727                 __be32 disn, sisn;
 728
 729                 if (sk->sk_state == TCP_NEW_SYN_RECV) {
 730                         struct request_sock *req = inet_reqsk(sk);
 731
 732                         sisn = htonl(tcp_rsk(req)->rcv_isn);
 733                         disn = htonl(tcp_rsk(req)->snt_isn);
 734                         *sne = tcp_ao_compute_sne(0, tcp_rsk(req)->snt_isn, seq);
 735                 } else {
 736                         sisn = th->seq;
 737                         disn = 0;
 738                 }
 739                 if (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6)
 740                         addr = (union tcp_md5_addr *)&ipv6_hdr(skb)->saddr;
 741                 else
 742                         addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
 743 #if IS_ENABLED(CONFIG_IPV6)
 744                 if (family == AF_INET6 && ipv6_addr_v4mapped(&sk->sk_v6_daddr))
 745                         family = AF_INET;
 746 #endif
 747
 748                 sk = sk_const_to_full_sk(sk);
 749                 ao_info = rcu_dereference(tcp_sk(sk)->ao_info);
 750                 if (!ao_info)
 751                         return -ENOENT;
 752                 *key = tcp_ao_do_lookup(sk, l3index, addr, family,
 753                                         -1, aoh->rnext_keyid);
 754                 if (!*key)
 755                         return -ENOENT;
 756                 *traffic_key = kmalloc(tcp_ao_digest_size(*key), GFP_ATOMIC);
 757                 if (!*traffic_key)
 758                         return -ENOMEM;
 759                 *allocated_traffic_key = true;
 760                 if (tcp_ao_calc_key_skb(*key, *traffic_key, skb,
 761                                         sisn, disn, family))
 762                         return -1;
 763                 *keyid = (*key)->rcvid;
 764         } else {
 765                 struct tcp_ao_key *rnext_key;
 766                 u32 snd_basis;
 767
 768                 if (sk->sk_state == TCP_TIME_WAIT) {
 769                         ao_info = rcu_dereference(tcp_twsk(sk)->ao_info);
 770                         snd_basis = tcp_twsk(sk)->tw_snd_nxt;
 771                 } else {
 772                         ao_info = rcu_dereference(tcp_sk(sk)->ao_info);
 773                         snd_basis = tcp_sk(sk)->snd_una;
 774                 }
 775                 if (!ao_info)
 776                         return -ENOENT;
 777
 778                 *key = tcp_ao_established_key(ao_info, aoh->rnext_keyid, -1);
 779                 if (!*key)
 780                         return -ENOENT;
 781                 *traffic_key = snd_other_key(*key);
 782                 rnext_key = READ_ONCE(ao_info->rnext_key);
 783                 *keyid = rnext_key->rcvid;
 784                 *sne = tcp_ao_compute_sne(READ_ONCE(ao_info->snd_sne),
 785                                           snd_basis, seq);
 786         }
 787         return 0;
 788 }
 789
 790 int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
 791                         struct tcp_ao_key *key, struct tcphdr *th,
 792                         __u8 *hash_location)
 793 {
 794         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 795         struct tcp_sock *tp = tcp_sk(sk);
 796         struct tcp_ao_info *ao;
 797         void *tkey_buf = NULL;
 798         u8 *traffic_key;
 799         u32 sne;
 800
 801         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
 802                                        lockdep_sock_is_held(sk));
 803         traffic_key = snd_other_key(key);
 804         if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
 805                 __be32 disn;
 806
 807                 if (!(tcb->tcp_flags & TCPHDR_ACK)) {
 808                         disn = 0;
 809                         tkey_buf = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
 810                         if (!tkey_buf)
 811                                 return -ENOMEM;
 812                         traffic_key = tkey_buf;
 813                 } else {
 814                         disn = ao->risn;
 815                 }
 816                 tp->af_specific->ao_calc_key_sk(key, traffic_key,
 817                                                 sk, ao->lisn, disn, true);
 818         }
 819         sne = tcp_ao_compute_sne(READ_ONCE(ao->snd_sne), READ_ONCE(tp->snd_una),
 820                                  ntohl(th->seq));
 821         tp->af_specific->calc_ao_hash(hash_location, key, sk, skb, traffic_key,
 822                                       hash_location - (u8 *)th, sne);
 823         kfree(tkey_buf);
 824         return 0;
 825 }
 826
 827 static struct tcp_ao_key *tcp_ao_inbound_lookup(unsigned short int family,
 828                 const struct sock *sk, const struct sk_buff *skb,
 829                 int sndid, int rcvid, int l3index)
 830 {
 831         if (family == AF_INET) {
 832                 const struct iphdr *iph = ip_hdr(skb);
 833
 834                 return tcp_ao_do_lookup(sk, l3index,
 835                                         (union tcp_ao_addr *)&iph->saddr,
 836                                         AF_INET, sndid, rcvid);
 837         } else {
 838                 const struct ipv6hdr *iph = ipv6_hdr(skb);
 839
 840                 return tcp_ao_do_lookup(sk, l3index,
 841                                         (union tcp_ao_addr *)&iph->saddr,
 842                                         AF_INET6, sndid, rcvid);
 843         }
 844 }
 845
 846 void tcp_ao_syncookie(struct sock *sk, const struct sk_buff *skb,
 847                       struct request_sock *req, unsigned short int family)
 848 {
 849         struct tcp_request_sock *treq = tcp_rsk(req);
 850         const struct tcphdr *th = tcp_hdr(skb);
 851         const struct tcp_ao_hdr *aoh;
 852         struct tcp_ao_key *key;
 853         int l3index;
 854
 855         /* treq->af_specific is used to perform TCP_AO lookup
 856          * in tcp_create_openreq_child().
 857          */
 858 #if IS_ENABLED(CONFIG_IPV6)
 859         if (family == AF_INET6)
 860                 treq->af_specific = &tcp_request_sock_ipv6_ops;
 861         else
 862 #endif
 863                 treq->af_specific = &tcp_request_sock_ipv4_ops;
 864
 865         treq->used_tcp_ao = false;
 866
 867         if (tcp_parse_auth_options(th, NULL, &aoh) || !aoh)
 868                 return;
 869
 870         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), inet_rsk(req)->ir_iif);
 871         key = tcp_ao_inbound_lookup(family, sk, skb, -1, aoh->keyid, l3index);
 872         if (!key)
 873                 /* Key not found, continue without TCP-AO */
 874                 return;
 875
 876         treq->ao_rcv_next = aoh->keyid;
 877         treq->ao_keyid = aoh->rnext_keyid;
 878         treq->used_tcp_ao = true;
 879 }
 880
 881 static enum skb_drop_reason
 882 tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb,
 883                    unsigned short int family, struct tcp_ao_info *info,
 884                    const struct tcp_ao_hdr *aoh, struct tcp_ao_key *key,
 885                    u8 *traffic_key, u8 *phash, u32 sne, int l3index)
 886 {
 887         u8 maclen = aoh->length - sizeof(struct tcp_ao_hdr);
 888         const struct tcphdr *th = tcp_hdr(skb);
 889         void *hash_buf = NULL;
 890
 891         if (maclen != tcp_ao_maclen(key)) {
 892                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
 893                 atomic64_inc(&info->counters.pkt_bad);
 894                 atomic64_inc(&key->pkt_bad);
 895                 tcp_hash_fail("AO hash wrong length", family, skb,
 896                               "%u != %d L3index: %d", maclen,
 897                               tcp_ao_maclen(key), l3index);
 898                 return SKB_DROP_REASON_TCP_AOFAILURE;
 899         }
 900
 901         hash_buf = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
 902         if (!hash_buf)
 903                 return SKB_DROP_REASON_NOT_SPECIFIED;
 904
 905         /* XXX: make it per-AF callback? */
 906         tcp_ao_hash_skb(family, hash_buf, key, sk, skb, traffic_key,
 907                         (phash - (u8 *)th), sne);
 908         if (memcmp(phash, hash_buf, maclen)) {
 909                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
 910                 atomic64_inc(&info->counters.pkt_bad);
 911                 atomic64_inc(&key->pkt_bad);
 912                 tcp_hash_fail("AO hash mismatch", family, skb,
 913                               "L3index: %d", l3index);
 914                 kfree(hash_buf);
 915                 return SKB_DROP_REASON_TCP_AOFAILURE;
 916         }
 917         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOGOOD);
 918         atomic64_inc(&info->counters.pkt_good);
 919         atomic64_inc(&key->pkt_good);
 920         kfree(hash_buf);
 921         return SKB_NOT_DROPPED_YET;
 922 }
 923
 924 enum skb_drop_reason
 925 tcp_inbound_ao_hash(struct sock *sk, const struct sk_buff *skb,
 926                     unsigned short int family, const struct request_sock *req,
 927                     int l3index, const struct tcp_ao_hdr *aoh)
 928 {
 929         const struct tcphdr *th = tcp_hdr(skb);
 930         u8 *phash = (u8 *)(aoh + 1); /* hash goes just after the header */
 931         struct tcp_ao_info *info;
 932         enum skb_drop_reason ret;
 933         struct tcp_ao_key *key;
 934         __be32 sisn, disn;
 935         u8 *traffic_key;
 936         u32 sne = 0;
 937
 938         info = rcu_dereference(tcp_sk(sk)->ao_info);
 939         if (!info) {
 940                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
 941                 tcp_hash_fail("AO key not found", family, skb,
 942                               "keyid: %u L3index: %d", aoh->keyid, l3index);
 943                 return SKB_DROP_REASON_TCP_AOUNEXPECTED;
 944         }
 945
 946         if (unlikely(th->syn)) {
 947                 sisn = th->seq;
 948                 disn = 0;
 949         }
 950
 951         /* Fast-path */
 952         if (likely((1 << sk->sk_state) & TCP_AO_ESTABLISHED)) {
 953                 enum skb_drop_reason err;
 954                 struct tcp_ao_key *current_key;
 955
 956                 /* Check if this socket's rnext_key matches the keyid in the
 957                  * packet. If not we lookup the key based on the keyid
 958                  * matching the rcvid in the mkt.
 959                  */
 960                 key = READ_ONCE(info->rnext_key);
 961                 if (key->rcvid != aoh->keyid) {
 962                         key = tcp_ao_established_key(info, -1, aoh->keyid);
 963                         if (!key)
 964                                 goto key_not_found;
 965                 }
 966
 967                 /* Delayed retransmitted SYN */
 968                 if (unlikely(th->syn && !th->ack))
 969                         goto verify_hash;
 970
 971                 sne = tcp_ao_compute_sne(info->rcv_sne, tcp_sk(sk)->rcv_nxt,
 972                                          ntohl(th->seq));
 973                 /* Established socket, traffic key are cached */
 974                 traffic_key = rcv_other_key(key);
 975                 err = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
 976                                          traffic_key, phash, sne, l3index);
 977                 if (err)
 978                         return err;
 979                 current_key = READ_ONCE(info->current_key);
 980                 /* Key rotation: the peer asks us to use new key (RNext) */
 981                 if (unlikely(aoh->rnext_keyid != current_key->sndid)) {
 982                         /* If the key is not found we do nothing. */
 983                         key = tcp_ao_established_key(info, aoh->rnext_keyid, -1);
 984                         if (key)
 985                                 /* pairs with tcp_ao_del_cmd */
 986                                 WRITE_ONCE(info->current_key, key);
 987                 }
 988                 return SKB_NOT_DROPPED_YET;
 989         }
 990
 991         /* Lookup key based on peer address and keyid.
 992          * current_key and rnext_key must not be used on tcp listen
 993          * sockets as otherwise:
 994          * - request sockets would race on those key pointers
 995          * - tcp_ao_del_cmd() allows async key removal
 996          */
 997         key = tcp_ao_inbound_lookup(family, sk, skb, -1, aoh->keyid, l3index);
 998         if (!key)
 999                 goto key_not_found;
1000
1001         if (th->syn && !th->ack)
1002                 goto verify_hash;
1003
1004         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV)) {
1005                 /* Make the initial syn the likely case here */
1006                 if (unlikely(req)) {
1007                         sne = tcp_ao_compute_sne(0, tcp_rsk(req)->rcv_isn,
1008                                                  ntohl(th->seq));
1009                         sisn = htonl(tcp_rsk(req)->rcv_isn);
1010                         disn = htonl(tcp_rsk(req)->snt_isn);
1011                 } else if (unlikely(th->ack && !th->syn)) {
1012                         /* Possible syncookie packet */
1013                         sisn = htonl(ntohl(th->seq) - 1);
1014                         disn = htonl(ntohl(th->ack_seq) - 1);
1015                         sne = tcp_ao_compute_sne(0, ntohl(sisn),
1016                                                  ntohl(th->seq));
1017                 } else if (unlikely(!th->syn)) {
1018                         /* no way to figure out initial sisn/disn - drop */
1019                         return SKB_DROP_REASON_TCP_FLAGS;
1020                 }
1021         } else if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1022                 disn = info->lisn;
1023                 if (th->syn || th->rst)
1024                         sisn = th->seq;
1025                 else
1026                         sisn = info->risn;
1027         } else {
1028                 WARN_ONCE(1, "TCP-AO: Unexpected sk_state %d", sk->sk_state);
1029                 return SKB_DROP_REASON_TCP_AOFAILURE;
1030         }
1031 verify_hash:
1032         traffic_key = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
1033         if (!traffic_key)
1034                 return SKB_DROP_REASON_NOT_SPECIFIED;
1035         tcp_ao_calc_key_skb(key, traffic_key, skb, sisn, disn, family);
1036         ret = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
1037                                  traffic_key, phash, sne, l3index);
1038         kfree(traffic_key);
1039         return ret;
1040
1041 key_not_found:
1042         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
1043         atomic64_inc(&info->counters.key_not_found);
1044         tcp_hash_fail("Requested by the peer AO key id not found",
1045                       family, skb, "L3index: %d", l3index);
1046         return SKB_DROP_REASON_TCP_AOKEYNOTFOUND;
1047 }
1048
1049 static int tcp_ao_cache_traffic_keys(const struct sock *sk,
1050                                      struct tcp_ao_info *ao,
1051                                      struct tcp_ao_key *ao_key)
1052 {
1053         u8 *traffic_key = snd_other_key(ao_key);
1054         int ret;
1055
1056         ret = tcp_ao_calc_key_sk(ao_key, traffic_key, sk,
1057                                  ao->lisn, ao->risn, true);
1058         if (ret)
1059                 return ret;
1060
1061         traffic_key = rcv_other_key(ao_key);
1062         ret = tcp_ao_calc_key_sk(ao_key, traffic_key, sk,
1063                                  ao->lisn, ao->risn, false);
1064         return ret;
1065 }
1066
1067 void tcp_ao_connect_init(struct sock *sk)
1068 {
1069         struct tcp_sock *tp = tcp_sk(sk);
1070         struct tcp_ao_info *ao_info;
1071         struct hlist_node *next;
1072         union tcp_ao_addr *addr;
1073         struct tcp_ao_key *key;
1074         int family, l3index;
1075
1076         ao_info = rcu_dereference_protected(tp->ao_info,
1077                                             lockdep_sock_is_held(sk));
1078         if (!ao_info)
1079                 return;
1080
1081         /* Remove all keys that don't match the peer */
1082         family = sk->sk_family;
1083         if (family == AF_INET)
1084                 addr = (union tcp_ao_addr *)&sk->sk_daddr;
1085 #if IS_ENABLED(CONFIG_IPV6)
1086         else if (family == AF_INET6)
1087                 addr = (union tcp_ao_addr *)&sk->sk_v6_daddr;
1088 #endif
1089         else
1090                 return;
1091         l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1092                                                  sk->sk_bound_dev_if);
1093
1094         hlist_for_each_entry_safe(key, next, &ao_info->head, node) {
1095                 if (!tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1))
1096                         continue;
1097
1098                 if (key == ao_info->current_key)
1099                         ao_info->current_key = NULL;
1100                 if (key == ao_info->rnext_key)
1101                         ao_info->rnext_key = NULL;
1102                 hlist_del_rcu(&key->node);
1103                 atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1104                 call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1105         }
1106
1107         key = tp->af_specific->ao_lookup(sk, sk, -1, -1);
1108         if (key) {
1109                 /* if current_key or rnext_key were not provided,
1110                  * use the first key matching the peer
1111                  */
1112                 if (!ao_info->current_key)
1113                         ao_info->current_key = key;
1114                 if (!ao_info->rnext_key)
1115                         ao_info->rnext_key = key;
1116                 tp->tcp_header_len += tcp_ao_len_aligned(key);
1117
1118                 ao_info->lisn = htonl(tp->write_seq);
1119                 ao_info->snd_sne = 0;
1120         } else {
1121                 /* Can't happen: tcp_connect() verifies that there's
1122                  * at least one tcp-ao key that matches the remote peer.
1123                  */
1124                 WARN_ON_ONCE(1);
1125                 rcu_assign_pointer(tp->ao_info, NULL);
1126                 kfree(ao_info);
1127         }
1128 }
1129
1130 void tcp_ao_established(struct sock *sk)
1131 {
1132         struct tcp_ao_info *ao;
1133         struct tcp_ao_key *key;
1134
1135         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
1136                                        lockdep_sock_is_held(sk));
1137         if (!ao)
1138                 return;
1139
1140         hlist_for_each_entry_rcu(key, &ao->head, node)
1141                 tcp_ao_cache_traffic_keys(sk, ao, key);
1142 }
1143
1144 void tcp_ao_finish_connect(struct sock *sk, struct sk_buff *skb)
1145 {
1146         struct tcp_ao_info *ao;
1147         struct tcp_ao_key *key;
1148
1149         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
1150                                        lockdep_sock_is_held(sk));
1151         if (!ao)
1152                 return;
1153
1154         WRITE_ONCE(ao->risn, tcp_hdr(skb)->seq);
1155         ao->rcv_sne = 0;
1156
1157         hlist_for_each_entry_rcu(key, &ao->head, node)
1158                 tcp_ao_cache_traffic_keys(sk, ao, key);
1159 }
1160
1161 int tcp_ao_copy_all_matching(const struct sock *sk, struct sock *newsk,
1162                              struct request_sock *req, struct sk_buff *skb,
1163                              int family)
1164 {
1165         struct tcp_ao_key *key, *new_key, *first_key;
1166         struct tcp_ao_info *new_ao, *ao;
1167         struct hlist_node *key_head;
1168         int l3index, ret = -ENOMEM;
1169         union tcp_ao_addr *addr;
1170         bool match = false;
1171
1172         ao = rcu_dereference(tcp_sk(sk)->ao_info);
1173         if (!ao)
1174                 return 0;
1175
1176         /* New socket without TCP-AO on it */
1177         if (!tcp_rsk_used_ao(req))
1178                 return 0;
1179
1180         new_ao = tcp_ao_alloc_info(GFP_ATOMIC);
1181         if (!new_ao)
1182                 return -ENOMEM;
1183         new_ao->lisn = htonl(tcp_rsk(req)->snt_isn);
1184         new_ao->risn = htonl(tcp_rsk(req)->rcv_isn);
1185         new_ao->ao_required = ao->ao_required;
1186         new_ao->accept_icmps = ao->accept_icmps;
1187
1188         if (family == AF_INET) {
1189                 addr = (union tcp_ao_addr *)&newsk->sk_daddr;
1190 #if IS_ENABLED(CONFIG_IPV6)
1191         } else if (family == AF_INET6) {
1192                 addr = (union tcp_ao_addr *)&newsk->sk_v6_daddr;
1193 #endif
1194         } else {
1195                 ret = -EAFNOSUPPORT;
1196                 goto free_ao;
1197         }
1198         l3index = l3mdev_master_ifindex_by_index(sock_net(newsk),
1199                                                  newsk->sk_bound_dev_if);
1200
1201         hlist_for_each_entry_rcu(key, &ao->head, node) {
1202                 if (tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1))
1203                         continue;
1204
1205                 new_key = tcp_ao_copy_key(newsk, key);
1206                 if (!new_key)
1207                         goto free_and_exit;
1208
1209                 tcp_ao_cache_traffic_keys(newsk, new_ao, new_key);
1210                 tcp_ao_link_mkt(new_ao, new_key);
1211                 match = true;
1212         }
1213
1214         if (!match) {
1215                 /* RFC5925 (7.4.1) specifies that the TCP-AO status
1216                  * of a connection is determined on the initial SYN.
1217                  * At this point the connection was TCP-AO enabled, so
1218                  * it can't switch to being unsigned if peer's key
1219                  * disappears on the listening socket.
1220                  */
1221                 ret = -EKEYREJECTED;
1222                 goto free_and_exit;
1223         }
1224
1225         if (!static_key_fast_inc_not_disabled(&tcp_ao_needed.key.key)) {
1226                 ret = -EUSERS;
1227                 goto free_and_exit;
1228         }
1229
1230         key_head = rcu_dereference(hlist_first_rcu(&new_ao->head));
1231         first_key = hlist_entry_safe(key_head, struct tcp_ao_key, node);
1232
1233         key = tcp_ao_established_key(new_ao, tcp_rsk(req)->ao_keyid, -1);
1234         if (key)
1235                 new_ao->current_key = key;
1236         else
1237                 new_ao->current_key = first_key;
1238
1239         /* set rnext_key */
1240         key = tcp_ao_established_key(new_ao, -1, tcp_rsk(req)->ao_rcv_next);
1241         if (key)
1242                 new_ao->rnext_key = key;
1243         else
1244                 new_ao->rnext_key = first_key;
1245
1246         sk_gso_disable(newsk);
1247         rcu_assign_pointer(tcp_sk(newsk)->ao_info, new_ao);
1248
1249         return 0;
1250
1251 free_and_exit:
1252         hlist_for_each_entry_safe(key, key_head, &new_ao->head, node) {
1253                 hlist_del(&key->node);
1254                 tcp_sigpool_release(key->tcp_sigpool_id);
1255                 atomic_sub(tcp_ao_sizeof_key(key), &newsk->sk_omem_alloc);
1256                 kfree_sensitive(key);
1257         }
1258 free_ao:
1259         kfree(new_ao);
1260         return ret;
1261 }
1262
1263 static bool tcp_ao_can_set_current_rnext(struct sock *sk)
1264 {
1265         /* There aren't current/rnext keys on TCP_LISTEN sockets */
1266         if (sk->sk_state == TCP_LISTEN)
1267                 return false;
1268         return true;
1269 }
1270
1271 static int tcp_ao_verify_ipv4(struct sock *sk, struct tcp_ao_add *cmd,
1272                               union tcp_ao_addr **addr)
1273 {
1274         struct sockaddr_in *sin = (struct sockaddr_in *)&cmd->addr;
1275         struct inet_sock *inet = inet_sk(sk);
1276
1277         if (sin->sin_family != AF_INET)
1278                 return -EINVAL;
1279
1280         /* Currently matching is not performed on port (or port ranges) */
1281         if (sin->sin_port != 0)
1282                 return -EINVAL;
1283
1284         /* Check prefix and trailing 0's in addr */
1285         if (cmd->prefix != 0) {
1286                 __be32 mask;
1287
1288                 if (ntohl(sin->sin_addr.s_addr) == INADDR_ANY)
1289                         return -EINVAL;
1290                 if (cmd->prefix > 32)
1291                         return -EINVAL;
1292
1293                 mask = inet_make_mask(cmd->prefix);
1294                 if (sin->sin_addr.s_addr & ~mask)
1295                         return -EINVAL;
1296
1297                 /* Check that MKT address is consistent with socket */
1298                 if (ntohl(inet->inet_daddr) != INADDR_ANY &&
1299                     (inet->inet_daddr & mask) != sin->sin_addr.s_addr)
1300                         return -EINVAL;
1301         } else {
1302                 if (ntohl(sin->sin_addr.s_addr) != INADDR_ANY)
1303                         return -EINVAL;
1304         }
1305
1306         *addr = (union tcp_ao_addr *)&sin->sin_addr;
1307         return 0;
1308 }
1309
1310 static int tcp_ao_parse_crypto(struct tcp_ao_add *cmd, struct tcp_ao_key *key)
1311 {
1312         unsigned int syn_tcp_option_space;
1313         bool is_kdf_aes_128_cmac = false;
1314         struct crypto_ahash *tfm;
1315         struct tcp_sigpool hp;
1316         void *tmp_key = NULL;
1317         int err;
1318
1319         /* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
1320         if (!strcmp("cmac(aes128)", cmd->alg_name)) {
1321                 strscpy(cmd->alg_name, "cmac(aes)", sizeof(cmd->alg_name));
1322                 is_kdf_aes_128_cmac = (cmd->keylen != 16);
1323                 tmp_key = kmalloc(cmd->keylen, GFP_KERNEL);
1324                 if (!tmp_key)
1325                         return -ENOMEM;
1326         }
1327
1328         key->maclen = cmd->maclen ?: 12; /* 12 is the default in RFC5925 */
1329
1330         /* Check: maclen + tcp-ao header <= (MAX_TCP_OPTION_SPACE - mss
1331          *                                      - tstamp (including sackperm)
1332          *                                      - wscale),
1333          * see tcp_syn_options(), tcp_synack_options(), commit 33ad798c924b.
1334          *
1335          * In order to allow D-SACK with TCP-AO, the header size should be:
1336          * (MAX_TCP_OPTION_SPACE - TCPOLEN_TSTAMP_ALIGNED
1337          *                      - TCPOLEN_SACK_BASE_ALIGNED
1338          *                      - 2 * TCPOLEN_SACK_PERBLOCK) = 8 (maclen = 4),
1339          * see tcp_established_options().
1340          *
1341          * RFC5925, 2.2:
1342          * Typical MACs are 96-128 bits (12-16 bytes), but any length
1343          * that fits in the header of the segment being authenticated
1344          * is allowed.
1345          *
1346          * RFC5925, 7.6:
1347          * TCP-AO continues to consume 16 bytes in non-SYN segments,
1348          * leaving a total of 24 bytes for other options, of which
1349          * the timestamp consumes 10.  This leaves 14 bytes, of which 10
1350          * are used for a single SACK block. When two SACK blocks are used,
1351          * such as to handle D-SACK, a smaller TCP-AO MAC would be required
1352          * to make room for the additional SACK block (i.e., to leave 18
1353          * bytes for the D-SACK variant of the SACK option) [RFC2883].
1354          * Note that D-SACK is not supportable in TCP MD5 in the presence
1355          * of timestamps, because TCP MD5’s MAC length is fixed and too
1356          * large to leave sufficient option space.
1357          */
1358         syn_tcp_option_space = MAX_TCP_OPTION_SPACE;
1359         syn_tcp_option_space -= TCPOLEN_MSS_ALIGNED;
1360         syn_tcp_option_space -= TCPOLEN_TSTAMP_ALIGNED;
1361         syn_tcp_option_space -= TCPOLEN_WSCALE_ALIGNED;
1362         if (tcp_ao_len_aligned(key) > syn_tcp_option_space) {
1363                 err = -EMSGSIZE;
1364                 goto err_kfree;
1365         }
1366
1367         key->keylen = cmd->keylen;
1368         memcpy(key->key, cmd->key, cmd->keylen);
1369
1370         err = tcp_sigpool_start(key->tcp_sigpool_id, &hp);
1371         if (err)
1372                 goto err_kfree;
1373
1374         tfm = crypto_ahash_reqtfm(hp.req);
1375         if (is_kdf_aes_128_cmac) {
1376                 void *scratch = hp.scratch;
1377                 struct scatterlist sg;
1378
1379                 memcpy(tmp_key, cmd->key, cmd->keylen);
1380                 sg_init_one(&sg, tmp_key, cmd->keylen);
1381
1382                 /* Using zero-key of 16 bytes as described in RFC5926 */
1383                 memset(scratch, 0, 16);
1384                 err = crypto_ahash_setkey(tfm, scratch, 16);
1385                 if (err)
1386                         goto err_pool_end;
1387
1388                 err = crypto_ahash_init(hp.req);
1389                 if (err)
1390                         goto err_pool_end;
1391
1392                 ahash_request_set_crypt(hp.req, &sg, key->key, cmd->keylen);
1393                 err = crypto_ahash_update(hp.req);
1394                 if (err)
1395                         goto err_pool_end;
1396
1397                 err |= crypto_ahash_final(hp.req);
1398                 if (err)
1399                         goto err_pool_end;
1400                 key->keylen = 16;
1401         }
1402
1403         err = crypto_ahash_setkey(tfm, key->key, key->keylen);
1404         if (err)
1405                 goto err_pool_end;
1406
1407         tcp_sigpool_end(&hp);
1408         kfree_sensitive(tmp_key);
1409
1410         if (tcp_ao_maclen(key) > key->digest_size)
1411                 return -EINVAL;
1412
1413         return 0;
1414
1415 err_pool_end:
1416         tcp_sigpool_end(&hp);
1417 err_kfree:
1418         kfree_sensitive(tmp_key);
1419         return err;
1420 }
1421
1422 #if IS_ENABLED(CONFIG_IPV6)
1423 static int tcp_ao_verify_ipv6(struct sock *sk, struct tcp_ao_add *cmd,
1424                               union tcp_ao_addr **paddr,
1425                               unsigned short int *family)
1426 {
1427         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd->addr;
1428         struct in6_addr *addr = &sin6->sin6_addr;
1429         u8 prefix = cmd->prefix;
1430
1431         if (sin6->sin6_family != AF_INET6)
1432                 return -EINVAL;
1433
1434         /* Currently matching is not performed on port (or port ranges) */
1435         if (sin6->sin6_port != 0)
1436                 return -EINVAL;
1437
1438         /* Check prefix and trailing 0's in addr */
1439         if (cmd->prefix != 0 && ipv6_addr_v4mapped(addr)) {
1440                 __be32 addr4 = addr->s6_addr32[3];
1441                 __be32 mask;
1442
1443                 if (prefix > 32 || ntohl(addr4) == INADDR_ANY)
1444                         return -EINVAL;
1445
1446                 mask = inet_make_mask(prefix);
1447                 if (addr4 & ~mask)
1448                         return -EINVAL;
1449
1450                 /* Check that MKT address is consistent with socket */
1451                 if (!ipv6_addr_any(&sk->sk_v6_daddr)) {
1452                         __be32 daddr4 = sk->sk_v6_daddr.s6_addr32[3];
1453
1454                         if (!ipv6_addr_v4mapped(&sk->sk_v6_daddr))
1455                                 return -EINVAL;
1456                         if ((daddr4 & mask) != addr4)
1457                                 return -EINVAL;
1458                 }
1459
1460                 *paddr = (union tcp_ao_addr *)&addr->s6_addr32[3];
1461                 *family = AF_INET;
1462                 return 0;
1463         } else if (cmd->prefix != 0) {
1464                 struct in6_addr pfx;
1465
1466                 if (ipv6_addr_any(addr) || prefix > 128)
1467                         return -EINVAL;
1468
1469                 ipv6_addr_prefix(&pfx, addr, prefix);
1470                 if (ipv6_addr_cmp(&pfx, addr))
1471                         return -EINVAL;
1472
1473                 /* Check that MKT address is consistent with socket */
1474                 if (!ipv6_addr_any(&sk->sk_v6_daddr) &&
1475                     !ipv6_prefix_equal(&sk->sk_v6_daddr, addr, prefix))
1476
1477                         return -EINVAL;
1478         } else {
1479                 if (!ipv6_addr_any(addr))
1480                         return -EINVAL;
1481         }
1482
1483         *paddr = (union tcp_ao_addr *)addr;
1484         return 0;
1485 }
1486 #else
1487 static int tcp_ao_verify_ipv6(struct sock *sk, struct tcp_ao_add *cmd,
1488                               union tcp_ao_addr **paddr,
1489                               unsigned short int *family)
1490 {
1491         return -EOPNOTSUPP;
1492 }
1493 #endif
1494
1495 static struct tcp_ao_info *setsockopt_ao_info(struct sock *sk)
1496 {
1497         if (sk_fullsock(sk)) {
1498                 return rcu_dereference_protected(tcp_sk(sk)->ao_info,
1499                                                  lockdep_sock_is_held(sk));
1500         } else if (sk->sk_state == TCP_TIME_WAIT) {
1501                 return rcu_dereference_protected(tcp_twsk(sk)->ao_info,
1502                                                  lockdep_sock_is_held(sk));
1503         }
1504         return ERR_PTR(-ESOCKTNOSUPPORT);
1505 }
1506
1507 static struct tcp_ao_info *getsockopt_ao_info(struct sock *sk)
1508 {
1509         if (sk_fullsock(sk))
1510                 return rcu_dereference(tcp_sk(sk)->ao_info);
1511         else if (sk->sk_state == TCP_TIME_WAIT)
1512                 return rcu_dereference(tcp_twsk(sk)->ao_info);
1513
1514         return ERR_PTR(-ESOCKTNOSUPPORT);
1515 }
1516
1517 #define TCP_AO_KEYF_ALL (TCP_AO_KEYF_IFINDEX | TCP_AO_KEYF_EXCLUDE_OPT)
1518 #define TCP_AO_GET_KEYF_VALID   (TCP_AO_KEYF_IFINDEX)
1519
1520 static struct tcp_ao_key *tcp_ao_key_alloc(struct sock *sk,
1521                                            struct tcp_ao_add *cmd)
1522 {
1523         const char *algo = cmd->alg_name;
1524         unsigned int digest_size;
1525         struct crypto_ahash *tfm;
1526         struct tcp_ao_key *key;
1527         struct tcp_sigpool hp;
1528         int err, pool_id;
1529         size_t size;
1530
1531         /* Force null-termination of alg_name */
1532         cmd->alg_name[ARRAY_SIZE(cmd->alg_name) - 1] = '\0';
1533
1534         /* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
1535         if (!strcmp("cmac(aes128)", algo))
1536                 algo = "cmac(aes)";
1537
1538         /* Full TCP header (th->doff << 2) should fit into scratch area,
1539          * see tcp_ao_hash_header().
1540          */
1541         pool_id = tcp_sigpool_alloc_ahash(algo, 60);
1542         if (pool_id < 0)
1543                 return ERR_PTR(pool_id);
1544
1545         err = tcp_sigpool_start(pool_id, &hp);
1546         if (err)
1547                 goto err_free_pool;
1548
1549         tfm = crypto_ahash_reqtfm(hp.req);
1550         digest_size = crypto_ahash_digestsize(tfm);
1551         tcp_sigpool_end(&hp);
1552
1553         size = sizeof(struct tcp_ao_key) + (digest_size << 1);
1554         key = sock_kmalloc(sk, size, GFP_KERNEL);
1555         if (!key) {
1556                 err = -ENOMEM;
1557                 goto err_free_pool;
1558         }
1559
1560         key->tcp_sigpool_id = pool_id;
1561         key->digest_size = digest_size;
1562         return key;
1563
1564 err_free_pool:
1565         tcp_sigpool_release(pool_id);
1566         return ERR_PTR(err);
1567 }
1568
1569 static int tcp_ao_add_cmd(struct sock *sk, unsigned short int family,
1570                           sockptr_t optval, int optlen)
1571 {
1572         struct tcp_ao_info *ao_info;
1573         union tcp_ao_addr *addr;
1574         struct tcp_ao_key *key;
1575         struct tcp_ao_add cmd;
1576         int ret, l3index = 0;
1577         bool first = false;
1578
1579         if (optlen < sizeof(cmd))
1580                 return -EINVAL;
1581
1582         ret = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1583         if (ret)
1584                 return ret;
1585
1586         if (cmd.keylen > TCP_AO_MAXKEYLEN)
1587                 return -EINVAL;
1588
1589         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1590                 return -EINVAL;
1591
1592         if (family == AF_INET)
1593                 ret = tcp_ao_verify_ipv4(sk, &cmd, &addr);
1594         else
1595                 ret = tcp_ao_verify_ipv6(sk, &cmd, &addr, &family);
1596         if (ret)
1597                 return ret;
1598
1599         if (cmd.keyflags & ~TCP_AO_KEYF_ALL)
1600                 return -EINVAL;
1601
1602         if (cmd.set_current || cmd.set_rnext) {
1603                 if (!tcp_ao_can_set_current_rnext(sk))
1604                         return -EINVAL;
1605         }
1606
1607         if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX))
1608                 return -EINVAL;
1609
1610         /* For cmd.tcp_ifindex = 0 the key will apply to the default VRF */
1611         if (cmd.keyflags & TCP_AO_KEYF_IFINDEX && cmd.ifindex) {
1612                 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
1613                 struct net_device *dev;
1614
1615                 rcu_read_lock();
1616                 dev = dev_get_by_index_rcu(sock_net(sk), cmd.ifindex);
1617                 if (dev && netif_is_l3_master(dev))
1618                         l3index = dev->ifindex;
1619                 rcu_read_unlock();
1620
1621                 if (!dev || !l3index)
1622                         return -EINVAL;
1623
1624                 if (!bound_dev_if || bound_dev_if != cmd.ifindex) {
1625                         /* tcp_ao_established_key() doesn't expect having
1626                          * non peer-matching key on an established TCP-AO
1627                          * connection.
1628                          */
1629                         if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1630                                 return -EINVAL;
1631                 }
1632
1633                 /* It's still possible to bind after adding keys or even
1634                  * re-bind to a different dev (with CAP_NET_RAW).
1635                  * So, no reason to return error here, rather try to be
1636                  * nice and warn the user.
1637                  */
1638                 if (bound_dev_if && bound_dev_if != cmd.ifindex)
1639                         net_warn_ratelimited("AO key ifindex %d != sk bound ifindex %d\n",
1640                                              cmd.ifindex, bound_dev_if);
1641         }
1642
1643         /* Don't allow keys for peers that have a matching TCP-MD5 key */
1644         if (cmd.keyflags & TCP_AO_KEYF_IFINDEX) {
1645                 /* Non-_exact version of tcp_md5_do_lookup() will
1646                  * as well match keys that aren't bound to a specific VRF
1647                  * (that will make them match AO key with
1648                  * sysctl_tcp_l3dev_accept = 1
1649                  */
1650                 if (tcp_md5_do_lookup(sk, l3index, addr, family))
1651                         return -EKEYREJECTED;
1652         } else {
1653                 if (tcp_md5_do_lookup_any_l3index(sk, addr, family))
1654                         return -EKEYREJECTED;
1655         }
1656
1657         ao_info = setsockopt_ao_info(sk);
1658         if (IS_ERR(ao_info))
1659                 return PTR_ERR(ao_info);
1660
1661         if (!ao_info) {
1662                 ao_info = tcp_ao_alloc_info(GFP_KERNEL);
1663                 if (!ao_info)
1664                         return -ENOMEM;
1665                 first = true;
1666         } else {
1667                 /* Check that neither RecvID nor SendID match any
1668                  * existing key for the peer, RFC5925 3.1:
1669                  * > The IDs of MKTs MUST NOT overlap where their
1670                  * > TCP connection identifiers overlap.
1671                  */
1672                 if (__tcp_ao_do_lookup(sk, l3index, addr, family, cmd.prefix, -1, cmd.rcvid))
1673                         return -EEXIST;
1674                 if (__tcp_ao_do_lookup(sk, l3index, addr, family,
1675                                        cmd.prefix, cmd.sndid, -1))
1676                         return -EEXIST;
1677         }
1678
1679         key = tcp_ao_key_alloc(sk, &cmd);
1680         if (IS_ERR(key)) {
1681                 ret = PTR_ERR(key);
1682                 goto err_free_ao;
1683         }
1684
1685         INIT_HLIST_NODE(&key->node);
1686         memcpy(&key->addr, addr, (family == AF_INET) ? sizeof(struct in_addr) :
1687                                                        sizeof(struct in6_addr));
1688         key->prefixlen  = cmd.prefix;
1689         key->family     = family;
1690         key->keyflags   = cmd.keyflags;
1691         key->sndid      = cmd.sndid;
1692         key->rcvid      = cmd.rcvid;
1693         key->l3index    = l3index;
1694         atomic64_set(&key->pkt_good, 0);
1695         atomic64_set(&key->pkt_bad, 0);
1696
1697         ret = tcp_ao_parse_crypto(&cmd, key);
1698         if (ret < 0)
1699                 goto err_free_sock;
1700
1701         if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))) {
1702                 tcp_ao_cache_traffic_keys(sk, ao_info, key);
1703                 if (first) {
1704                         ao_info->current_key = key;
1705                         ao_info->rnext_key = key;
1706                 }
1707         }
1708
1709         tcp_ao_link_mkt(ao_info, key);
1710         if (first) {
1711                 if (!static_branch_inc(&tcp_ao_needed.key)) {
1712                         ret = -EUSERS;
1713                         goto err_free_sock;
1714                 }
1715                 sk_gso_disable(sk);
1716                 rcu_assign_pointer(tcp_sk(sk)->ao_info, ao_info);
1717         }
1718
1719         if (cmd.set_current)
1720                 WRITE_ONCE(ao_info->current_key, key);
1721         if (cmd.set_rnext)
1722                 WRITE_ONCE(ao_info->rnext_key, key);
1723         return 0;
1724
1725 err_free_sock:
1726         atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1727         tcp_sigpool_release(key->tcp_sigpool_id);
1728         kfree_sensitive(key);
1729 err_free_ao:
1730         if (first)
1731                 kfree(ao_info);
1732         return ret;
1733 }
1734
1735 static int tcp_ao_delete_key(struct sock *sk, struct tcp_ao_info *ao_info,
1736                              bool del_async, struct tcp_ao_key *key,
1737                              struct tcp_ao_key *new_current,
1738                              struct tcp_ao_key *new_rnext)
1739 {
1740         int err;
1741
1742         hlist_del_rcu(&key->node);
1743
1744         /* Support for async delete on listening sockets: as they don't
1745          * need current_key/rnext_key maintaining, we don't need to check
1746          * them and we can just free all resources in RCU fashion.
1747          */
1748         if (del_async) {
1749                 atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1750                 call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1751                 return 0;
1752         }
1753
1754         /* At this moment another CPU could have looked this key up
1755          * while it was unlinked from the list. Wait for RCU grace period,
1756          * after which the key is off-list and can't be looked up again;
1757          * the rx path [just before RCU came] might have used it and set it
1758          * as current_key (very unlikely).
1759          * Free the key with next RCU grace period (in case it was
1760          * current_key before tcp_ao_current_rnext() might have
1761          * changed it in forced-delete).
1762          */
1763         synchronize_rcu();
1764         if (new_current)
1765                 WRITE_ONCE(ao_info->current_key, new_current);
1766         if (new_rnext)
1767                 WRITE_ONCE(ao_info->rnext_key, new_rnext);
1768
1769         if (unlikely(READ_ONCE(ao_info->current_key) == key ||
1770                      READ_ONCE(ao_info->rnext_key) == key)) {
1771                 err = -EBUSY;
1772                 goto add_key;
1773         }
1774
1775         atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1776         call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1777
1778         return 0;
1779 add_key:
1780         hlist_add_head_rcu(&key->node, &ao_info->head);
1781         return err;
1782 }
1783
1784 #define TCP_AO_DEL_KEYF_ALL (TCP_AO_KEYF_IFINDEX)
1785 static int tcp_ao_del_cmd(struct sock *sk, unsigned short int family,
1786                           sockptr_t optval, int optlen)
1787 {
1788         struct tcp_ao_key *key, *new_current = NULL, *new_rnext = NULL;
1789         int err, addr_len, l3index = 0;
1790         struct tcp_ao_info *ao_info;
1791         union tcp_ao_addr *addr;
1792         struct tcp_ao_del cmd;
1793         __u8 prefix;
1794         u16 port;
1795
1796         if (optlen < sizeof(cmd))
1797                 return -EINVAL;
1798
1799         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1800         if (err)
1801                 return err;
1802
1803         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1804                 return -EINVAL;
1805
1806         if (cmd.set_current || cmd.set_rnext) {
1807                 if (!tcp_ao_can_set_current_rnext(sk))
1808                         return -EINVAL;
1809         }
1810
1811         if (cmd.keyflags & ~TCP_AO_DEL_KEYF_ALL)
1812                 return -EINVAL;
1813
1814         /* No sanity check for TCP_AO_KEYF_IFINDEX as if a VRF
1815          * was destroyed, there still should be a way to delete keys,
1816          * that were bound to that l3intf. So, fail late at lookup stage
1817          * if there is no key for that ifindex.
1818          */
1819         if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX))
1820                 return -EINVAL;
1821
1822         ao_info = setsockopt_ao_info(sk);
1823         if (IS_ERR(ao_info))
1824                 return PTR_ERR(ao_info);
1825         if (!ao_info)
1826                 return -ENOENT;
1827
1828         /* For sockets in TCP_CLOSED it's possible set keys that aren't
1829          * matching the future peer (address/VRF/etc),
1830          * tcp_ao_connect_init() will choose a correct matching MKT
1831          * if there's any.
1832          */
1833         if (cmd.set_current) {
1834                 new_current = tcp_ao_established_key(ao_info, cmd.current_key, -1);
1835                 if (!new_current)
1836                         return -ENOENT;
1837         }
1838         if (cmd.set_rnext) {
1839                 new_rnext = tcp_ao_established_key(ao_info, -1, cmd.rnext);
1840                 if (!new_rnext)
1841                         return -ENOENT;
1842         }
1843         if (cmd.del_async && sk->sk_state != TCP_LISTEN)
1844                 return -EINVAL;
1845
1846         if (family == AF_INET) {
1847                 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.addr;
1848
1849                 addr = (union tcp_ao_addr *)&sin->sin_addr;
1850                 addr_len = sizeof(struct in_addr);
1851                 port = ntohs(sin->sin_port);
1852         } else {
1853                 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.addr;
1854                 struct in6_addr *addr6 = &sin6->sin6_addr;
1855
1856                 if (ipv6_addr_v4mapped(addr6)) {
1857                         addr = (union tcp_ao_addr *)&addr6->s6_addr32[3];
1858                         addr_len = sizeof(struct in_addr);
1859                         family = AF_INET;
1860                 } else {
1861                         addr = (union tcp_ao_addr *)addr6;
1862                         addr_len = sizeof(struct in6_addr);
1863                 }
1864                 port = ntohs(sin6->sin6_port);
1865         }
1866         prefix = cmd.prefix;
1867
1868         /* Currently matching is not performed on port (or port ranges) */
1869         if (port != 0)
1870                 return -EINVAL;
1871
1872         /* We could choose random present key here for current/rnext
1873          * but that's less predictable. Let's be strict and don't
1874          * allow removing a key that's in use. RFC5925 doesn't
1875          * specify how-to coordinate key removal, but says:
1876          * "It is presumed that an MKT affecting a particular
1877          * connection cannot be destroyed during an active connection"
1878          */
1879         hlist_for_each_entry_rcu(key, &ao_info->head, node) {
1880                 if (cmd.sndid != key->sndid ||
1881                     cmd.rcvid != key->rcvid)
1882                         continue;
1883
1884                 if (family != key->family ||
1885                     prefix != key->prefixlen ||
1886                     memcmp(addr, &key->addr, addr_len))
1887                         continue;
1888
1889                 if ((cmd.keyflags & TCP_AO_KEYF_IFINDEX) !=
1890                     (key->keyflags & TCP_AO_KEYF_IFINDEX))
1891                         continue;
1892
1893                 if (key->l3index != l3index)
1894                         continue;
1895
1896                 if (key == new_current || key == new_rnext)
1897                         continue;
1898
1899                 return tcp_ao_delete_key(sk, ao_info, cmd.del_async, key,
1900                                          new_current, new_rnext);
1901         }
1902         return -ENOENT;
1903 }
1904
1905 /* cmd.ao_required makes a socket TCP-AO only.
1906  * Don't allow any md5 keys for any l3intf on the socket together with it.
1907  * Restricting it early in setsockopt() removes a check for
1908  * ao_info->ao_required on inbound tcp segment fast-path.
1909  */
1910 static int tcp_ao_required_verify(struct sock *sk)
1911 {
1912 #ifdef CONFIG_TCP_MD5SIG
1913         const struct tcp_md5sig_info *md5sig;
1914
1915         if (!static_branch_unlikely(&tcp_md5_needed.key))
1916                 return 0;
1917
1918         md5sig = rcu_dereference_check(tcp_sk(sk)->md5sig_info,
1919                                        lockdep_sock_is_held(sk));
1920         if (!md5sig)
1921                 return 0;
1922
1923         if (rcu_dereference_check(hlist_first_rcu(&md5sig->head),
1924                                   lockdep_sock_is_held(sk)))
1925                 return 1;
1926 #endif
1927         return 0;
1928 }
1929
1930 static int tcp_ao_info_cmd(struct sock *sk, unsigned short int family,
1931                            sockptr_t optval, int optlen)
1932 {
1933         struct tcp_ao_key *new_current = NULL, *new_rnext = NULL;
1934         struct tcp_ao_info *ao_info;
1935         struct tcp_ao_info_opt cmd;
1936         bool first = false;
1937         int err;
1938
1939         if (optlen < sizeof(cmd))
1940                 return -EINVAL;
1941
1942         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1943         if (err)
1944                 return err;
1945
1946         if (cmd.set_current || cmd.set_rnext) {
1947                 if (!tcp_ao_can_set_current_rnext(sk))
1948                         return -EINVAL;
1949         }
1950
1951         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1952                 return -EINVAL;
1953
1954         ao_info = setsockopt_ao_info(sk);
1955         if (IS_ERR(ao_info))
1956                 return PTR_ERR(ao_info);
1957         if (!ao_info) {
1958                 if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1959                         return -EINVAL;
1960                 ao_info = tcp_ao_alloc_info(GFP_KERNEL);
1961                 if (!ao_info)
1962                         return -ENOMEM;
1963                 first = true;
1964         }
1965
1966         if (cmd.ao_required && tcp_ao_required_verify(sk))
1967                 return -EKEYREJECTED;
1968
1969         /* For sockets in TCP_CLOSED it's possible set keys that aren't
1970          * matching the future peer (address/port/VRF/etc),
1971          * tcp_ao_connect_init() will choose a correct matching MKT
1972          * if there's any.
1973          */
1974         if (cmd.set_current) {
1975                 new_current = tcp_ao_established_key(ao_info, cmd.current_key, -1);
1976                 if (!new_current) {
1977                         err = -ENOENT;
1978                         goto out;
1979                 }
1980         }
1981         if (cmd.set_rnext) {
1982                 new_rnext = tcp_ao_established_key(ao_info, -1, cmd.rnext);
1983                 if (!new_rnext) {
1984                         err = -ENOENT;
1985                         goto out;
1986                 }
1987         }
1988         if (cmd.set_counters) {
1989                 atomic64_set(&ao_info->counters.pkt_good, cmd.pkt_good);
1990                 atomic64_set(&ao_info->counters.pkt_bad, cmd.pkt_bad);
1991                 atomic64_set(&ao_info->counters.key_not_found, cmd.pkt_key_not_found);
1992                 atomic64_set(&ao_info->counters.ao_required, cmd.pkt_ao_required);
1993                 atomic64_set(&ao_info->counters.dropped_icmp, cmd.pkt_dropped_icmp);
1994         }
1995
1996         ao_info->ao_required = cmd.ao_required;
1997         ao_info->accept_icmps = cmd.accept_icmps;
1998         if (new_current)
1999                 WRITE_ONCE(ao_info->current_key, new_current);
2000         if (new_rnext)
2001                 WRITE_ONCE(ao_info->rnext_key, new_rnext);
2002         if (first) {
2003                 if (!static_branch_inc(&tcp_ao_needed.key)) {
2004                         err = -EUSERS;
2005                         goto out;
2006                 }
2007                 sk_gso_disable(sk);
2008                 rcu_assign_pointer(tcp_sk(sk)->ao_info, ao_info);
2009         }
2010         return 0;
2011 out:
2012         if (first)
2013                 kfree(ao_info);
2014         return err;
2015 }
2016
2017 int tcp_parse_ao(struct sock *sk, int cmd, unsigned short int family,
2018                  sockptr_t optval, int optlen)
2019 {
2020         if (WARN_ON_ONCE(family != AF_INET && family != AF_INET6))
2021                 return -EAFNOSUPPORT;
2022
2023         switch (cmd) {
2024         case TCP_AO_ADD_KEY:
2025                 return tcp_ao_add_cmd(sk, family, optval, optlen);
2026         case TCP_AO_DEL_KEY:
2027                 return tcp_ao_del_cmd(sk, family, optval, optlen);
2028         case TCP_AO_INFO:
2029                 return tcp_ao_info_cmd(sk, family, optval, optlen);
2030         default:
2031                 WARN_ON_ONCE(1);
2032                 return -EINVAL;
2033         }
2034 }
2035
2036 int tcp_v4_parse_ao(struct sock *sk, int cmd, sockptr_t optval, int optlen)
2037 {
2038         return tcp_parse_ao(sk, cmd, AF_INET, optval, optlen);
2039 }
2040
2041 /* tcp_ao_copy_mkts_to_user(ao_info, optval, optlen)
2042  *
2043  * @ao_info:    struct tcp_ao_info on the socket that
2044  *              socket getsockopt(TCP_AO_GET_KEYS) is executed on
2045  * @optval:     pointer to array of tcp_ao_getsockopt structures in user space.
2046  *              Must be != NULL.
2047  * @optlen:     pointer to size of tcp_ao_getsockopt structure.
2048  *              Must be != NULL.
2049  *
2050  * Return value: 0 on success, a negative error number otherwise.
2051  *
2052  * optval points to an array of tcp_ao_getsockopt structures in user space.
2053  * optval[0] is used as both input and output to getsockopt. It determines
2054  * which keys are returned by the kernel.
2055  * optval[0].nkeys is the size of the array in user space. On return it contains
2056  * the number of keys matching the search criteria.
2057  * If tcp_ao_getsockopt::get_all is set, then all keys in the socket are
2058  * returned, otherwise only keys matching <addr, prefix, sndid, rcvid>
2059  * in optval[0] are returned.
2060  * optlen is also used as both input and output. The user provides the size
2061  * of struct tcp_ao_getsockopt in user space, and the kernel returns the size
2062  * of the structure in kernel space.
2063  * The size of struct tcp_ao_getsockopt may differ between user and kernel.
2064  * There are three cases to consider:
2065  *  * If usize == ksize, then keys are copied verbatim.
2066  *  * If usize < ksize, then the userspace has passed an old struct to a
2067  *    newer kernel. The rest of the trailing bytes in optval[0]
2068  *    (ksize - usize) are interpreted as 0 by the kernel.
2069  *  * If usize > ksize, then the userspace has passed a new struct to an
2070  *    older kernel. The trailing bytes unknown to the kernel (usize - ksize)
2071  *    are checked to ensure they are zeroed, otherwise -E2BIG is returned.
2072  * On return the kernel fills in min(usize, ksize) in each entry of the array.
2073  * The layout of the fields in the user and kernel structures is expected to
2074  * be the same (including in the 32bit vs 64bit case).
2075  */
2076 static int tcp_ao_copy_mkts_to_user(struct tcp_ao_info *ao_info,
2077                                     sockptr_t optval, sockptr_t optlen)
2078 {
2079         struct tcp_ao_getsockopt opt_in, opt_out;
2080         struct tcp_ao_key *key, *current_key;
2081         bool do_address_matching = true;
2082         union tcp_ao_addr *addr = NULL;
2083         int err, l3index, user_len;
2084         unsigned int max_keys;  /* maximum number of keys to copy to user */
2085         size_t out_offset = 0;
2086         size_t bytes_to_write;  /* number of bytes to write to user level */
2087         u32 matched_keys;       /* keys from ao_info matched so far */
2088         int optlen_out;
2089         __be16 port = 0;
2090
2091         if (copy_from_sockptr(&user_len, optlen, sizeof(int)))
2092                 return -EFAULT;
2093
2094         if (user_len <= 0)
2095                 return -EINVAL;
2096
2097         memset(&opt_in, 0, sizeof(struct tcp_ao_getsockopt));
2098         err = copy_struct_from_sockptr(&opt_in, sizeof(opt_in),
2099                                        optval, user_len);
2100         if (err < 0)
2101                 return err;
2102
2103         if (opt_in.pkt_good || opt_in.pkt_bad)
2104                 return -EINVAL;
2105         if (opt_in.keyflags & ~TCP_AO_GET_KEYF_VALID)
2106                 return -EINVAL;
2107         if (opt_in.ifindex && !(opt_in.keyflags & TCP_AO_KEYF_IFINDEX))
2108                 return -EINVAL;
2109
2110         if (opt_in.reserved != 0)
2111                 return -EINVAL;
2112
2113         max_keys = opt_in.nkeys;
2114         l3index = (opt_in.keyflags & TCP_AO_KEYF_IFINDEX) ? opt_in.ifindex : -1;
2115
2116         if (opt_in.get_all || opt_in.is_current || opt_in.is_rnext) {
2117                 if (opt_in.get_all && (opt_in.is_current || opt_in.is_rnext))
2118                         return -EINVAL;
2119                 do_address_matching = false;
2120         }
2121
2122         switch (opt_in.addr.ss_family) {
2123         case AF_INET: {
2124                 struct sockaddr_in *sin;
2125                 __be32 mask;
2126
2127                 sin = (struct sockaddr_in *)&opt_in.addr;
2128                 port = sin->sin_port;
2129                 addr = (union tcp_ao_addr *)&sin->sin_addr;
2130
2131                 if (opt_in.prefix > 32)
2132                         return -EINVAL;
2133
2134                 if (ntohl(sin->sin_addr.s_addr) == INADDR_ANY &&
2135                     opt_in.prefix != 0)
2136                         return -EINVAL;
2137
2138                 mask = inet_make_mask(opt_in.prefix);
2139                 if (sin->sin_addr.s_addr & ~mask)
2140                         return -EINVAL;
2141
2142                 break;
2143         }
2144         case AF_INET6: {
2145                 struct sockaddr_in6 *sin6;
2146                 struct in6_addr *addr6;
2147
2148                 sin6 = (struct sockaddr_in6 *)&opt_in.addr;
2149                 addr = (union tcp_ao_addr *)&sin6->sin6_addr;
2150                 addr6 = &sin6->sin6_addr;
2151                 port = sin6->sin6_port;
2152
2153                 /* We don't have to change family and @addr here if
2154                  * ipv6_addr_v4mapped() like in key adding:
2155                  * tcp_ao_key_cmp() does it. Do the sanity checks though.
2156                  */
2157                 if (opt_in.prefix != 0) {
2158                         if (ipv6_addr_v4mapped(addr6)) {
2159                                 __be32 mask, addr4 = addr6->s6_addr32[3];
2160
2161                                 if (opt_in.prefix > 32 ||
2162                                     ntohl(addr4) == INADDR_ANY)
2163                                         return -EINVAL;
2164                                 mask = inet_make_mask(opt_in.prefix);
2165                                 if (addr4 & ~mask)
2166                                         return -EINVAL;
2167                         } else {
2168                                 struct in6_addr pfx;
2169
2170                                 if (ipv6_addr_any(addr6) ||
2171                                     opt_in.prefix > 128)
2172                                         return -EINVAL;
2173
2174                                 ipv6_addr_prefix(&pfx, addr6, opt_in.prefix);
2175                                 if (ipv6_addr_cmp(&pfx, addr6))
2176                                         return -EINVAL;
2177                         }
2178                 } else if (!ipv6_addr_any(addr6)) {
2179                         return -EINVAL;
2180                 }
2181                 break;
2182         }
2183         case 0:
2184                 if (!do_address_matching)
2185                         break;
2186                 fallthrough;
2187         default:
2188                 return -EAFNOSUPPORT;
2189         }
2190
2191         if (!do_address_matching) {
2192                 /* We could just ignore those, but let's do stricter checks */
2193                 if (addr || port)
2194                         return -EINVAL;
2195                 if (opt_in.prefix || opt_in.sndid || opt_in.rcvid)
2196                         return -EINVAL;
2197         }
2198
2199         bytes_to_write = min_t(int, user_len, sizeof(struct tcp_ao_getsockopt));
2200         matched_keys = 0;
2201         /* May change in RX, while we're dumping, pre-fetch it */
2202         current_key = READ_ONCE(ao_info->current_key);
2203
2204         hlist_for_each_entry_rcu(key, &ao_info->head, node) {
2205                 if (opt_in.get_all)
2206                         goto match;
2207
2208                 if (opt_in.is_current || opt_in.is_rnext) {
2209                         if (opt_in.is_current && key == current_key)
2210                                 goto match;
2211                         if (opt_in.is_rnext && key == ao_info->rnext_key)
2212                                 goto match;
2213                         continue;
2214                 }
2215
2216                 if (tcp_ao_key_cmp(key, l3index, addr, opt_in.prefix,
2217                                    opt_in.addr.ss_family,
2218                                    opt_in.sndid, opt_in.rcvid) != 0)
2219                         continue;
2220 match:
2221                 matched_keys++;
2222                 if (matched_keys > max_keys)
2223                         continue;
2224
2225                 memset(&opt_out, 0, sizeof(struct tcp_ao_getsockopt));
2226
2227                 if (key->family == AF_INET) {
2228                         struct sockaddr_in *sin_out = (struct sockaddr_in *)&opt_out.addr;
2229
2230                         sin_out->sin_family = key->family;
2231                         sin_out->sin_port = 0;
2232                         memcpy(&sin_out->sin_addr, &key->addr, sizeof(struct in_addr));
2233                 } else {
2234                         struct sockaddr_in6 *sin6_out = (struct sockaddr_in6 *)&opt_out.addr;
2235
2236                         sin6_out->sin6_family = key->family;
2237                         sin6_out->sin6_port = 0;
2238                         memcpy(&sin6_out->sin6_addr, &key->addr, sizeof(struct in6_addr));
2239                 }
2240                 opt_out.sndid = key->sndid;
2241                 opt_out.rcvid = key->rcvid;
2242                 opt_out.prefix = key->prefixlen;
2243                 opt_out.keyflags = key->keyflags;
2244                 opt_out.is_current = (key == current_key);
2245                 opt_out.is_rnext = (key == ao_info->rnext_key);
2246                 opt_out.nkeys = 0;
2247                 opt_out.maclen = key->maclen;
2248                 opt_out.keylen = key->keylen;
2249                 opt_out.ifindex = key->l3index;
2250                 opt_out.pkt_good = atomic64_read(&key->pkt_good);
2251                 opt_out.pkt_bad = atomic64_read(&key->pkt_bad);
2252                 memcpy(&opt_out.key, key->key, key->keylen);
2253                 tcp_sigpool_algo(key->tcp_sigpool_id, opt_out.alg_name, 64);
2254
2255                 /* Copy key to user */
2256                 if (copy_to_sockptr_offset(optval, out_offset,
2257                                            &opt_out, bytes_to_write))
2258                         return -EFAULT;
2259                 out_offset += user_len;
2260         }
2261
2262         optlen_out = (int)sizeof(struct tcp_ao_getsockopt);
2263         if (copy_to_sockptr(optlen, &optlen_out, sizeof(int)))
2264                 return -EFAULT;
2265
2266         out_offset = offsetof(struct tcp_ao_getsockopt, nkeys);
2267         if (copy_to_sockptr_offset(optval, out_offset,
2268                                    &matched_keys, sizeof(u32)))
2269                 return -EFAULT;
2270
2271         return 0;
2272 }
2273
2274 int tcp_ao_get_mkts(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2275 {
2276         struct tcp_ao_info *ao_info;
2277
2278         ao_info = setsockopt_ao_info(sk);
2279         if (IS_ERR(ao_info))
2280                 return PTR_ERR(ao_info);
2281         if (!ao_info)
2282                 return -ENOENT;
2283
2284         return tcp_ao_copy_mkts_to_user(ao_info, optval, optlen);
2285 }
2286
2287 int tcp_ao_get_sock_info(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2288 {
2289         struct tcp_ao_info_opt out, in = {};
2290         struct tcp_ao_key *current_key;
2291         struct tcp_ao_info *ao;
2292         int err, len;
2293
2294         if (copy_from_sockptr(&len, optlen, sizeof(int)))
2295                 return -EFAULT;
2296
2297         if (len <= 0)
2298                 return -EINVAL;
2299
2300         /* Copying this "in" only to check ::reserved, ::reserved2,
2301          * that may be needed to extend (struct tcp_ao_info_opt) and
2302          * what getsockopt() provides in future.
2303          */
2304         err = copy_struct_from_sockptr(&in, sizeof(in), optval, len);
2305         if (err)
2306                 return err;
2307
2308         if (in.reserved != 0 || in.reserved2 != 0)
2309                 return -EINVAL;
2310
2311         ao = setsockopt_ao_info(sk);
2312         if (IS_ERR(ao))
2313                 return PTR_ERR(ao);
2314         if (!ao)
2315                 return -ENOENT;
2316
2317         memset(&out, 0, sizeof(out));
2318         out.ao_required         = ao->ao_required;
2319         out.accept_icmps        = ao->accept_icmps;
2320         out.pkt_good            = atomic64_read(&ao->counters.pkt_good);
2321         out.pkt_bad             = atomic64_read(&ao->counters.pkt_bad);
2322         out.pkt_key_not_found   = atomic64_read(&ao->counters.key_not_found);
2323         out.pkt_ao_required     = atomic64_read(&ao->counters.ao_required);
2324         out.pkt_dropped_icmp    = atomic64_read(&ao->counters.dropped_icmp);
2325
2326         current_key = READ_ONCE(ao->current_key);
2327         if (current_key) {
2328                 out.set_current = 1;
2329                 out.current_key = current_key->sndid;
2330         }
2331         if (ao->rnext_key) {
2332                 out.set_rnext = 1;
2333                 out.rnext = ao->rnext_key->rcvid;
2334         }
2335
2336         if (copy_to_sockptr(optval, &out, min_t(int, len, sizeof(out))))
2337                 return -EFAULT;
2338
2339         return 0;
2340 }
2341
2342 int tcp_ao_set_repair(struct sock *sk, sockptr_t optval, unsigned int optlen)
2343 {
2344         struct tcp_sock *tp = tcp_sk(sk);
2345         struct tcp_ao_repair cmd;
2346         struct tcp_ao_key *key;
2347         struct tcp_ao_info *ao;
2348         int err;
2349
2350         if (optlen < sizeof(cmd))
2351                 return -EINVAL;
2352
2353         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
2354         if (err)
2355                 return err;
2356
2357         if (!tp->repair)
2358                 return -EPERM;
2359
2360         ao = setsockopt_ao_info(sk);
2361         if (IS_ERR(ao))
2362                 return PTR_ERR(ao);
2363         if (!ao)
2364                 return -ENOENT;
2365
2366         WRITE_ONCE(ao->lisn, cmd.snt_isn);
2367         WRITE_ONCE(ao->risn, cmd.rcv_isn);
2368         WRITE_ONCE(ao->snd_sne, cmd.snd_sne);
2369         WRITE_ONCE(ao->rcv_sne, cmd.rcv_sne);
2370
2371         hlist_for_each_entry_rcu(key, &ao->head, node)
2372                 tcp_ao_cache_traffic_keys(sk, ao, key);
2373
2374         return 0;
2375 }
2376
2377 int tcp_ao_get_repair(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2378 {
2379         struct tcp_sock *tp = tcp_sk(sk);
2380         struct tcp_ao_repair opt;
2381         struct tcp_ao_info *ao;
2382         int len;
2383
2384         if (copy_from_sockptr(&len, optlen, sizeof(int)))
2385                 return -EFAULT;
2386
2387         if (len <= 0)
2388                 return -EINVAL;
2389
2390         if (!tp->repair)
2391                 return -EPERM;
2392
2393         rcu_read_lock();
2394         ao = getsockopt_ao_info(sk);
2395         if (IS_ERR_OR_NULL(ao)) {
2396                 rcu_read_unlock();
2397                 return ao ? PTR_ERR(ao) : -ENOENT;
2398         }
2399
2400         opt.snt_isn     = ao->lisn;
2401         opt.rcv_isn     = ao->risn;
2402         opt.snd_sne     = READ_ONCE(ao->snd_sne);
2403         opt.rcv_sne     = READ_ONCE(ao->rcv_sne);
2404         rcu_read_unlock();
2405
2406         if (copy_to_sockptr(optval, &opt, min_t(int, len, sizeof(opt))))
2407                 return -EFAULT;
2408         return 0;
2409 }