drivers/staging/lustre/lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lnet/selftest/conctl.c
  33  *
  34  * Infrastructure of LST console
  35  *
  36  * Author: Liang Zhen <liangzhen@clusterfs.com>
  37  */
  38
  39 #include "../../include/linux/libcfs/libcfs.h"
  40 #include "../../include/linux/lnet/lib-lnet.h"
  41 #include "console.h"
  42 #include "conrpc.h"
  43
  44 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  45 do {                                                    \
  46         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  47                 (p)->nle_nactive++;                     \
  48         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  49                 (p)->nle_nbusy++;                       \
  50         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  51                 (p)->nle_ndown++;                       \
  52         else                                            \
  53                 (p)->nle_nunknown++;                    \
  54         (p)->nle_nnode++;                               \
  55 } while (0)
  56
  57 struct lstcon_session console_session;
  58
  59 static void
  60 lstcon_node_get(struct lstcon_node *nd)
  61 {
  62         LASSERT(nd->nd_ref >= 1);
  63
  64         nd->nd_ref++;
  65 }
  66
  67 static int
  68 lstcon_node_find(lnet_process_id_t id, struct lstcon_node **ndpp, int create)
  69 {
  70         struct lstcon_ndlink    *ndl;
  71         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  72
  73         LASSERT(id.nid != LNET_NID_ANY);
  74
  75         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
  76                             ndl_hlink) {
  77                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  78                     ndl->ndl_node->nd_id.pid != id.pid)
  79                         continue;
  80
  81                 lstcon_node_get(ndl->ndl_node);
  82                 *ndpp = ndl->ndl_node;
  83                 return 0;
  84         }
  85
  86         if (!create)
  87                 return -ENOENT;
  88
  89         LIBCFS_ALLOC(*ndpp, sizeof(struct lstcon_node) + sizeof(struct lstcon_ndlink));
  90         if (!*ndpp)
  91                 return -ENOMEM;
  92
  93         ndl = (struct lstcon_ndlink *)(*ndpp + 1);
  94
  95         ndl->ndl_node = *ndpp;
  96
  97         ndl->ndl_node->nd_ref = 1;
  98         ndl->ndl_node->nd_id = id;
  99         ndl->ndl_node->nd_stamp = cfs_time_current();
 100         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 101         ndl->ndl_node->nd_timeout = 0;
 102         memset(&ndl->ndl_node->nd_ping, 0, sizeof(struct lstcon_rpc));
 103
 104         /*
 105          * queued in global hash & list, no refcount is taken by
 106          * global hash & list, if caller release his refcount,
 107          * node will be released
 108          */
 109         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 110         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 111
 112         return 0;
 113 }
 114
 115 static void
 116 lstcon_node_put(struct lstcon_node *nd)
 117 {
 118         struct lstcon_ndlink *ndl;
 119
 120         LASSERT(nd->nd_ref > 0);
 121
 122         if (--nd->nd_ref > 0)
 123                 return;
 124
 125         ndl = (struct lstcon_ndlink *)(nd + 1);
 126
 127         LASSERT(!list_empty(&ndl->ndl_link));
 128         LASSERT(!list_empty(&ndl->ndl_hlink));
 129
 130         /* remove from session */
 131         list_del(&ndl->ndl_link);
 132         list_del(&ndl->ndl_hlink);
 133
 134         LIBCFS_FREE(nd, sizeof(struct lstcon_node) + sizeof(struct lstcon_ndlink));
 135 }
 136
 137 static int
 138 lstcon_ndlink_find(struct list_head *hash,
 139                    lnet_process_id_t id, struct lstcon_ndlink **ndlpp, int create)
 140 {
 141         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 142         struct lstcon_ndlink *ndl;
 143         struct lstcon_node *nd;
 144         int rc;
 145
 146         if (id.nid == LNET_NID_ANY)
 147                 return -EINVAL;
 148
 149         /* search in hash */
 150         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 151                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 152                     ndl->ndl_node->nd_id.pid != id.pid)
 153                         continue;
 154
 155                 *ndlpp = ndl;
 156                 return 0;
 157         }
 158
 159         if (!create)
 160                 return -ENOENT;
 161
 162         /* find or create in session hash */
 163         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 164         if (rc)
 165                 return rc;
 166
 167         LIBCFS_ALLOC(ndl, sizeof(struct lstcon_ndlink));
 168         if (!ndl) {
 169                 lstcon_node_put(nd);
 170                 return -ENOMEM;
 171         }
 172
 173         *ndlpp = ndl;
 174
 175         ndl->ndl_node = nd;
 176         INIT_LIST_HEAD(&ndl->ndl_link);
 177         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 178
 179         return 0;
 180 }
 181
 182 static void
 183 lstcon_ndlink_release(struct lstcon_ndlink *ndl)
 184 {
 185         LASSERT(list_empty(&ndl->ndl_link));
 186         LASSERT(!list_empty(&ndl->ndl_hlink));
 187
 188         list_del(&ndl->ndl_hlink); /* delete from hash */
 189         lstcon_node_put(ndl->ndl_node);
 190
 191         LIBCFS_FREE(ndl, sizeof(*ndl));
 192 }
 193
 194 static int
 195 lstcon_group_alloc(char *name, struct lstcon_group **grpp)
 196 {
 197         struct lstcon_group *grp;
 198         int i;
 199
 200         LIBCFS_ALLOC(grp, offsetof(struct lstcon_group,
 201                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 202         if (!grp)
 203                 return -ENOMEM;
 204
 205         grp->grp_ref = 1;
 206         if (name) {
 207                 if (strlen(name) > sizeof(grp->grp_name) - 1) {
 208                         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 209                                     grp_ndl_hash[LST_NODE_HASHSIZE]));
 210                         return -E2BIG;
 211                 }
 212                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
 213         }
 214
 215         INIT_LIST_HEAD(&grp->grp_link);
 216         INIT_LIST_HEAD(&grp->grp_ndl_list);
 217         INIT_LIST_HEAD(&grp->grp_trans_list);
 218
 219         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 220                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 221
 222         *grpp = grp;
 223
 224         return 0;
 225 }
 226
 227 static void
 228 lstcon_group_addref(struct lstcon_group *grp)
 229 {
 230         grp->grp_ref++;
 231 }
 232
 233 static void lstcon_group_ndlink_release(struct lstcon_group *, struct lstcon_ndlink *);
 234
 235 static void
 236 lstcon_group_drain(struct lstcon_group *grp, int keep)
 237 {
 238         struct lstcon_ndlink *ndl;
 239         struct lstcon_ndlink *tmp;
 240
 241         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 242                 if (!(ndl->ndl_node->nd_state & keep))
 243                         lstcon_group_ndlink_release(grp, ndl);
 244         }
 245 }
 246
 247 static void
 248 lstcon_group_decref(struct lstcon_group *grp)
 249 {
 250         int i;
 251
 252         if (--grp->grp_ref > 0)
 253                 return;
 254
 255         if (!list_empty(&grp->grp_link))
 256                 list_del(&grp->grp_link);
 257
 258         lstcon_group_drain(grp, 0);
 259
 260         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 261                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 262
 263         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 264                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 265 }
 266
 267 static int
 268 lstcon_group_find(const char *name, struct lstcon_group **grpp)
 269 {
 270         struct lstcon_group *grp;
 271
 272         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 273                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE))
 274                         continue;
 275
 276                 lstcon_group_addref(grp); /* +1 ref for caller */
 277                 *grpp = grp;
 278                 return 0;
 279         }
 280
 281         return -ENOENT;
 282 }
 283
 284 static int
 285 lstcon_group_ndlink_find(struct lstcon_group *grp, lnet_process_id_t id,
 286                          struct lstcon_ndlink **ndlpp, int create)
 287 {
 288         int rc;
 289
 290         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 291         if (rc)
 292                 return rc;
 293
 294         if (!list_empty(&(*ndlpp)->ndl_link))
 295                 return 0;
 296
 297         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 298         grp->grp_nnode++;
 299
 300         return 0;
 301 }
 302
 303 static void
 304 lstcon_group_ndlink_release(struct lstcon_group *grp, struct lstcon_ndlink *ndl)
 305 {
 306         list_del_init(&ndl->ndl_link);
 307         lstcon_ndlink_release(ndl);
 308         grp->grp_nnode--;
 309 }
 310
 311 static void
 312 lstcon_group_ndlink_move(struct lstcon_group *old,
 313                          struct lstcon_group *new, struct lstcon_ndlink *ndl)
 314 {
 315         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 316                                         LST_NODE_HASHSIZE;
 317
 318         list_del(&ndl->ndl_hlink);
 319         list_del(&ndl->ndl_link);
 320         old->grp_nnode--;
 321
 322         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 323         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 324         new->grp_nnode++;
 325 }
 326
 327 static void
 328 lstcon_group_move(struct lstcon_group *old, struct lstcon_group *new)
 329 {
 330         struct lstcon_ndlink *ndl;
 331
 332         while (!list_empty(&old->grp_ndl_list)) {
 333                 ndl = list_entry(old->grp_ndl_list.next,
 334                                  struct lstcon_ndlink, ndl_link);
 335                 lstcon_group_ndlink_move(old, new, ndl);
 336         }
 337 }
 338
 339 static int
 340 lstcon_sesrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 341 {
 342         struct lstcon_group *grp = (struct lstcon_group *)arg;
 343
 344         switch (transop) {
 345         case LST_TRANS_SESNEW:
 346                 if (nd->nd_state == LST_NODE_ACTIVE)
 347                         return 0;
 348                 break;
 349
 350         case LST_TRANS_SESEND:
 351                 if (nd->nd_state != LST_NODE_ACTIVE)
 352                         return 0;
 353
 354                 if (grp && nd->nd_ref > 1)
 355                         return 0;
 356                 break;
 357
 358         case LST_TRANS_SESQRY:
 359                 break;
 360
 361         default:
 362                 LBUG();
 363         }
 364
 365         return 1;
 366 }
 367
 368 static int
 369 lstcon_sesrpc_readent(int transop, struct srpc_msg *msg,
 370                       lstcon_rpc_ent_t __user *ent_up)
 371 {
 372         struct srpc_debug_reply *rep;
 373
 374         switch (transop) {
 375         case LST_TRANS_SESNEW:
 376         case LST_TRANS_SESEND:
 377                 return 0;
 378
 379         case LST_TRANS_SESQRY:
 380                 rep = &msg->msg_body.dbg_reply;
 381
 382                 if (copy_to_user(&ent_up->rpe_priv[0],
 383                                  &rep->dbg_timeout, sizeof(int)) ||
 384                     copy_to_user(&ent_up->rpe_payload[0],
 385                                  &rep->dbg_name, LST_NAME_SIZE))
 386                         return -EFAULT;
 387
 388                 return 0;
 389
 390         default:
 391                 LBUG();
 392         }
 393
 394         return 0;
 395 }
 396
 397 static int
 398 lstcon_group_nodes_add(struct lstcon_group *grp,
 399                        int count, lnet_process_id_t __user *ids_up,
 400                        unsigned *featp, struct list_head __user *result_up)
 401 {
 402         struct lstcon_rpc_trans *trans;
 403         struct lstcon_ndlink    *ndl;
 404         struct lstcon_group *tmp;
 405         lnet_process_id_t id;
 406         int i;
 407         int rc;
 408
 409         rc = lstcon_group_alloc(NULL, &tmp);
 410         if (rc) {
 411                 CERROR("Out of memory\n");
 412                 return -ENOMEM;
 413         }
 414
 415         for (i = 0 ; i < count; i++) {
 416                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 417                         rc = -EFAULT;
 418                         break;
 419                 }
 420
 421                 /* skip if it's in this group already */
 422                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 423                 if (!rc)
 424                         continue;
 425
 426                 /* add to tmp group */
 427                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 428                 if (rc) {
 429                         CERROR("Can't create ndlink, out of memory\n");
 430                         break;
 431                 }
 432         }
 433
 434         if (rc) {
 435                 lstcon_group_decref(tmp);
 436                 return rc;
 437         }
 438
 439         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 440                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 441                                      tmp, lstcon_sesrpc_condition, &trans);
 442         if (rc) {
 443                 CERROR("Can't create transaction: %d\n", rc);
 444                 lstcon_group_decref(tmp);
 445                 return rc;
 446         }
 447
 448         /* post all RPCs */
 449         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 450
 451         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 452                                           lstcon_sesrpc_readent);
 453         *featp = trans->tas_features;
 454
 455         /* destroy all RPGs */
 456         lstcon_rpc_trans_destroy(trans);
 457
 458         lstcon_group_move(tmp, grp);
 459         lstcon_group_decref(tmp);
 460
 461         return rc;
 462 }
 463
 464 static int
 465 lstcon_group_nodes_remove(struct lstcon_group *grp,
 466                           int count, lnet_process_id_t __user *ids_up,
 467                           struct list_head __user *result_up)
 468 {
 469         struct lstcon_rpc_trans *trans;
 470         struct lstcon_ndlink *ndl;
 471         struct lstcon_group *tmp;
 472         lnet_process_id_t id;
 473         int rc;
 474         int i;
 475
 476         /* End session and remove node from the group */
 477
 478         rc = lstcon_group_alloc(NULL, &tmp);
 479         if (rc) {
 480                 CERROR("Out of memory\n");
 481                 return -ENOMEM;
 482         }
 483
 484         for (i = 0; i < count; i++) {
 485                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 486                         rc = -EFAULT;
 487                         goto error;
 488                 }
 489
 490                 /* move node to tmp group */
 491                 if (!lstcon_group_ndlink_find(grp, id, &ndl, 0))
 492                         lstcon_group_ndlink_move(grp, tmp, ndl);
 493         }
 494
 495         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 496                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 497                                      tmp, lstcon_sesrpc_condition, &trans);
 498         if (rc) {
 499                 CERROR("Can't create transaction: %d\n", rc);
 500                 goto error;
 501         }
 502
 503         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 504
 505         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 506
 507         lstcon_rpc_trans_destroy(trans);
 508         /* release nodes anyway, because we can't rollback status */
 509         lstcon_group_decref(tmp);
 510
 511         return rc;
 512 error:
 513         lstcon_group_move(tmp, grp);
 514         lstcon_group_decref(tmp);
 515
 516         return rc;
 517 }
 518
 519 int
 520 lstcon_group_add(char *name)
 521 {
 522         struct lstcon_group *grp;
 523         int rc;
 524
 525         rc = lstcon_group_find(name, &grp) ? 0 : -EEXIST;
 526         if (rc) {
 527                 /* find a group with same name */
 528                 lstcon_group_decref(grp);
 529                 return rc;
 530         }
 531
 532         rc = lstcon_group_alloc(name, &grp);
 533         if (rc) {
 534                 CERROR("Can't allocate descriptor for group %s\n", name);
 535                 return -ENOMEM;
 536         }
 537
 538         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 539
 540         return rc;
 541 }
 542
 543 int
 544 lstcon_nodes_add(char *name, int count, lnet_process_id_t __user *ids_up,
 545                  unsigned *featp, struct list_head __user *result_up)
 546 {
 547         struct lstcon_group *grp;
 548         int rc;
 549
 550         LASSERT(count > 0);
 551         LASSERT(ids_up);
 552
 553         rc = lstcon_group_find(name, &grp);
 554         if (rc) {
 555                 CDEBUG(D_NET, "Can't find group %s\n", name);
 556                 return rc;
 557         }
 558
 559         if (grp->grp_ref > 2) {
 560                 /* referred by other threads or test */
 561                 CDEBUG(D_NET, "Group %s is busy\n", name);
 562                 lstcon_group_decref(grp);
 563
 564                 return -EBUSY;
 565         }
 566
 567         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 568
 569         lstcon_group_decref(grp);
 570
 571         return rc;
 572 }
 573
 574 int
 575 lstcon_group_del(char *name)
 576 {
 577         struct lstcon_rpc_trans *trans;
 578         struct lstcon_group *grp;
 579         int rc;
 580
 581         rc = lstcon_group_find(name, &grp);
 582         if (rc) {
 583                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 584                 return rc;
 585         }
 586
 587         if (grp->grp_ref > 2) {
 588                 /* referred by others threads or test */
 589                 CDEBUG(D_NET, "Group %s is busy\n", name);
 590                 lstcon_group_decref(grp);
 591                 return -EBUSY;
 592         }
 593
 594         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 595                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 596                                      grp, lstcon_sesrpc_condition, &trans);
 597         if (rc) {
 598                 CERROR("Can't create transaction: %d\n", rc);
 599                 lstcon_group_decref(grp);
 600                 return rc;
 601         }
 602
 603         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 604
 605         lstcon_rpc_trans_destroy(trans);
 606
 607         lstcon_group_decref(grp);
 608         /*
 609          * -ref for session, it's destroyed,
 610          * status can't be rolled back, destroy group anyway
 611          */
 612         lstcon_group_decref(grp);
 613
 614         return rc;
 615 }
 616
 617 int
 618 lstcon_group_clean(char *name, int args)
 619 {
 620         struct lstcon_group *grp = NULL;
 621         int rc;
 622
 623         rc = lstcon_group_find(name, &grp);
 624         if (rc) {
 625                 CDEBUG(D_NET, "Can't find group %s\n", name);
 626                 return rc;
 627         }
 628
 629         if (grp->grp_ref > 2) {
 630                 /* referred by test */
 631                 CDEBUG(D_NET, "Group %s is busy\n", name);
 632                 lstcon_group_decref(grp);
 633                 return -EBUSY;
 634         }
 635
 636         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 637                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 638
 639         lstcon_group_drain(grp, args);
 640
 641         lstcon_group_decref(grp);
 642         /* release empty group */
 643         if (list_empty(&grp->grp_ndl_list))
 644                 lstcon_group_decref(grp);
 645
 646         return 0;
 647 }
 648
 649 int
 650 lstcon_nodes_remove(char *name, int count, lnet_process_id_t __user *ids_up,
 651                     struct list_head __user *result_up)
 652 {
 653         struct lstcon_group *grp = NULL;
 654         int rc;
 655
 656         rc = lstcon_group_find(name, &grp);
 657         if (rc) {
 658                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 659                 return rc;
 660         }
 661
 662         if (grp->grp_ref > 2) {
 663                 /* referred by test */
 664                 CDEBUG(D_NET, "Group %s is busy\n", name);
 665                 lstcon_group_decref(grp);
 666                 return -EBUSY;
 667         }
 668
 669         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 670
 671         lstcon_group_decref(grp);
 672         /* release empty group */
 673         if (list_empty(&grp->grp_ndl_list))
 674                 lstcon_group_decref(grp);
 675
 676         return rc;
 677 }
 678
 679 int
 680 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 681 {
 682         struct lstcon_rpc_trans *trans;
 683         struct lstcon_group *grp;
 684         int rc;
 685
 686         rc = lstcon_group_find(name, &grp);
 687         if (rc) {
 688                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 689                 return rc;
 690         }
 691
 692         if (grp->grp_ref > 2) {
 693                 /* referred by test */
 694                 CDEBUG(D_NET, "Group %s is busy\n", name);
 695                 lstcon_group_decref(grp);
 696                 return -EBUSY;
 697         }
 698
 699         /* re-invite all inactive nodes int the group */
 700         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 701                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 702                                      grp, lstcon_sesrpc_condition, &trans);
 703         if (rc) {
 704                 /* local error, return */
 705                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 706                 lstcon_group_decref(grp);
 707                 return rc;
 708         }
 709
 710         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 711
 712         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 713
 714         lstcon_rpc_trans_destroy(trans);
 715         /* -ref for me */
 716         lstcon_group_decref(grp);
 717
 718         return rc;
 719 }
 720
 721 int
 722 lstcon_group_list(int index, int len, char __user *name_up)
 723 {
 724         struct lstcon_group *grp;
 725
 726         LASSERT(index >= 0);
 727         LASSERT(name_up);
 728
 729         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 730                 if (!index--) {
 731                         return copy_to_user(name_up, grp->grp_name, len) ?
 732                                             -EFAULT : 0;
 733                 }
 734         }
 735
 736         return -ENOENT;
 737 }
 738
 739 static int
 740 lstcon_nodes_getent(struct list_head *head, int *index_p,
 741                     int *count_p, lstcon_node_ent_t __user *dents_up)
 742 {
 743         struct lstcon_ndlink *ndl;
 744         struct lstcon_node *nd;
 745         int count = 0;
 746         int index = 0;
 747
 748         LASSERT(index_p && count_p);
 749         LASSERT(dents_up);
 750         LASSERT(*index_p >= 0);
 751         LASSERT(*count_p > 0);
 752
 753         list_for_each_entry(ndl, head, ndl_link) {
 754                 if (index++ < *index_p)
 755                         continue;
 756
 757                 if (count >= *count_p)
 758                         break;
 759
 760                 nd = ndl->ndl_node;
 761                 if (copy_to_user(&dents_up[count].nde_id,
 762                                  &nd->nd_id, sizeof(nd->nd_id)) ||
 763                     copy_to_user(&dents_up[count].nde_state,
 764                                  &nd->nd_state, sizeof(nd->nd_state)))
 765                         return -EFAULT;
 766
 767                 count++;
 768         }
 769
 770         if (index <= *index_p)
 771                 return -ENOENT;
 772
 773         *count_p = count;
 774         *index_p = index;
 775
 776         return 0;
 777 }
 778
 779 int
 780 lstcon_group_info(char *name, lstcon_ndlist_ent_t __user *gents_p,
 781                   int *index_p, int *count_p,
 782                   lstcon_node_ent_t __user *dents_up)
 783 {
 784         lstcon_ndlist_ent_t *gentp;
 785         struct lstcon_group *grp;
 786         struct lstcon_ndlink *ndl;
 787         int rc;
 788
 789         rc = lstcon_group_find(name, &grp);
 790         if (rc) {
 791                 CDEBUG(D_NET, "Can't find group %s\n", name);
 792                 return rc;
 793         }
 794
 795         if (dents_up) {
 796                 /* verbose query */
 797                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 798                                          index_p, count_p, dents_up);
 799                 lstcon_group_decref(grp);
 800
 801                 return rc;
 802         }
 803
 804         /* non-verbose query */
 805         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 806         if (!gentp) {
 807                 CERROR("Can't allocate ndlist_ent\n");
 808                 lstcon_group_decref(grp);
 809
 810                 return -ENOMEM;
 811         }
 812
 813         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 814                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 815
 816         rc = copy_to_user(gents_p, gentp,
 817                           sizeof(lstcon_ndlist_ent_t)) ? -EFAULT : 0;
 818
 819         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 820
 821         lstcon_group_decref(grp);
 822
 823         return 0;
 824 }
 825
 826 static int
 827 lstcon_batch_find(const char *name, struct lstcon_batch **batpp)
 828 {
 829         struct lstcon_batch *bat;
 830
 831         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 832                 if (!strncmp(bat->bat_name, name, LST_NAME_SIZE)) {
 833                         *batpp = bat;
 834                         return 0;
 835                 }
 836         }
 837
 838         return -ENOENT;
 839 }
 840
 841 int
 842 lstcon_batch_add(char *name)
 843 {
 844         struct lstcon_batch *bat;
 845         int i;
 846         int rc;
 847
 848         rc = !lstcon_batch_find(name, &bat) ? -EEXIST : 0;
 849         if (rc) {
 850                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 851                 return rc;
 852         }
 853
 854         LIBCFS_ALLOC(bat, sizeof(struct lstcon_batch));
 855         if (!bat) {
 856                 CERROR("Can't allocate descriptor for batch %s\n", name);
 857                 return -ENOMEM;
 858         }
 859
 860         LIBCFS_ALLOC(bat->bat_cli_hash,
 861                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 862         if (!bat->bat_cli_hash) {
 863                 CERROR("Can't allocate hash for batch %s\n", name);
 864                 LIBCFS_FREE(bat, sizeof(struct lstcon_batch));
 865
 866                 return -ENOMEM;
 867         }
 868
 869         LIBCFS_ALLOC(bat->bat_srv_hash,
 870                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 871         if (!bat->bat_srv_hash) {
 872                 CERROR("Can't allocate hash for batch %s\n", name);
 873                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 874                 LIBCFS_FREE(bat, sizeof(struct lstcon_batch));
 875
 876                 return -ENOMEM;
 877         }
 878
 879         if (strlen(name) > sizeof(bat->bat_name) - 1) {
 880                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 881                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 882                 LIBCFS_FREE(bat, sizeof(struct lstcon_batch));
 883                 return -E2BIG;
 884         }
 885         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
 886         bat->bat_hdr.tsb_index = 0;
 887         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 888
 889         bat->bat_ntest = 0;
 890         bat->bat_state = LST_BATCH_IDLE;
 891
 892         INIT_LIST_HEAD(&bat->bat_cli_list);
 893         INIT_LIST_HEAD(&bat->bat_srv_list);
 894         INIT_LIST_HEAD(&bat->bat_test_list);
 895         INIT_LIST_HEAD(&bat->bat_trans_list);
 896
 897         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 898                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 899                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 900         }
 901
 902         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 903
 904         return rc;
 905 }
 906
 907 int
 908 lstcon_batch_list(int index, int len, char __user *name_up)
 909 {
 910         struct lstcon_batch *bat;
 911
 912         LASSERT(name_up);
 913         LASSERT(index >= 0);
 914
 915         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 916                 if (!index--) {
 917                         return copy_to_user(name_up, bat->bat_name, len) ?
 918                                             -EFAULT : 0;
 919                 }
 920         }
 921
 922         return -ENOENT;
 923 }
 924
 925 int
 926 lstcon_batch_info(char *name, lstcon_test_batch_ent_t __user *ent_up,
 927                   int server, int testidx, int *index_p, int *ndent_p,
 928                   lstcon_node_ent_t __user *dents_up)
 929 {
 930         lstcon_test_batch_ent_t *entp;
 931         struct list_head *clilst;
 932         struct list_head *srvlst;
 933         struct lstcon_test *test = NULL;
 934         struct lstcon_batch *bat;
 935         struct lstcon_ndlink    *ndl;
 936         int rc;
 937
 938         rc = lstcon_batch_find(name, &bat);
 939         if (rc) {
 940                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 941                 return -ENOENT;
 942         }
 943
 944         if (testidx > 0) {
 945                 /* query test, test index start from 1 */
 946                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 947                         if (testidx-- == 1)
 948                                 break;
 949                 }
 950
 951                 if (testidx > 0) {
 952                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 953                         return -ENOENT;
 954                 }
 955         }
 956
 957         clilst = !test ? &bat->bat_cli_list :
 958                          &test->tes_src_grp->grp_ndl_list;
 959         srvlst = !test ? &bat->bat_srv_list :
 960                          &test->tes_dst_grp->grp_ndl_list;
 961
 962         if (dents_up) {
 963                 rc = lstcon_nodes_getent((server ? srvlst : clilst),
 964                                          index_p, ndent_p, dents_up);
 965                 return rc;
 966         }
 967
 968         /* non-verbose query */
 969         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 970         if (!entp)
 971                 return -ENOMEM;
 972
 973         if (!test) {
 974                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 975                 entp->u.tbe_batch.bae_state = bat->bat_state;
 976         } else {
 977                 entp->u.tbe_test.tse_type = test->tes_type;
 978                 entp->u.tbe_test.tse_loop = test->tes_loop;
 979                 entp->u.tbe_test.tse_concur = test->tes_concur;
 980         }
 981
 982         list_for_each_entry(ndl, clilst, ndl_link)
 983                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 984
 985         list_for_each_entry(ndl, srvlst, ndl_link)
 986                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 987
 988         rc = copy_to_user(ent_up, entp,
 989                           sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 990
 991         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 992
 993         return rc;
 994 }
 995
 996 static int
 997 lstcon_batrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 998 {
 999         switch (transop) {
1000         case LST_TRANS_TSBRUN:
1001                 if (nd->nd_state != LST_NODE_ACTIVE)
1002                         return -ENETDOWN;
1003                 break;
1004
1005         case LST_TRANS_TSBSTOP:
1006                 if (nd->nd_state != LST_NODE_ACTIVE)
1007                         return 0;
1008                 break;
1009
1010         case LST_TRANS_TSBCLIQRY:
1011         case LST_TRANS_TSBSRVQRY:
1012                 break;
1013         }
1014
1015         return 1;
1016 }
1017
1018 static int
1019 lstcon_batch_op(struct lstcon_batch *bat, int transop,
1020                 struct list_head __user *result_up)
1021 {
1022         struct lstcon_rpc_trans *trans;
1023         int rc;
1024
1025         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1026                                      &bat->bat_trans_list, transop,
1027                                      bat, lstcon_batrpc_condition, &trans);
1028         if (rc) {
1029                 CERROR("Can't create transaction: %d\n", rc);
1030                 return rc;
1031         }
1032
1033         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1034
1035         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1036
1037         lstcon_rpc_trans_destroy(trans);
1038
1039         return rc;
1040 }
1041
1042 int
1043 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1044 {
1045         struct lstcon_batch *bat;
1046         int rc;
1047
1048         if (lstcon_batch_find(name, &bat)) {
1049                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1050                 return -ENOENT;
1051         }
1052
1053         bat->bat_arg = timeout;
1054
1055         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1056
1057         /* mark batch as running if it's started in any node */
1058         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0))
1059                 bat->bat_state = LST_BATCH_RUNNING;
1060
1061         return rc;
1062 }
1063
1064 int
1065 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1066 {
1067         struct lstcon_batch *bat;
1068         int rc;
1069
1070         if (lstcon_batch_find(name, &bat)) {
1071                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1072                 return -ENOENT;
1073         }
1074
1075         bat->bat_arg = force;
1076
1077         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1078
1079         /* mark batch as stopped if all RPCs finished */
1080         if (!lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0))
1081                 bat->bat_state = LST_BATCH_IDLE;
1082
1083         return rc;
1084 }
1085
1086 static void
1087 lstcon_batch_destroy(struct lstcon_batch *bat)
1088 {
1089         struct lstcon_ndlink *ndl;
1090         struct lstcon_test *test;
1091         int i;
1092
1093         list_del(&bat->bat_link);
1094
1095         while (!list_empty(&bat->bat_test_list)) {
1096                 test = list_entry(bat->bat_test_list.next,
1097                                   struct lstcon_test, tes_link);
1098                 LASSERT(list_empty(&test->tes_trans_list));
1099
1100                 list_del(&test->tes_link);
1101
1102                 lstcon_group_decref(test->tes_src_grp);
1103                 lstcon_group_decref(test->tes_dst_grp);
1104
1105                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1106                                            tes_param[test->tes_paramlen]));
1107         }
1108
1109         LASSERT(list_empty(&bat->bat_trans_list));
1110
1111         while (!list_empty(&bat->bat_cli_list)) {
1112                 ndl = list_entry(bat->bat_cli_list.next,
1113                                  struct lstcon_ndlink, ndl_link);
1114                 list_del_init(&ndl->ndl_link);
1115
1116                 lstcon_ndlink_release(ndl);
1117         }
1118
1119         while (!list_empty(&bat->bat_srv_list)) {
1120                 ndl = list_entry(bat->bat_srv_list.next,
1121                                  struct lstcon_ndlink, ndl_link);
1122                 list_del_init(&ndl->ndl_link);
1123
1124                 lstcon_ndlink_release(ndl);
1125         }
1126
1127         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1128                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1129                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1130         }
1131
1132         LIBCFS_FREE(bat->bat_cli_hash,
1133                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1134         LIBCFS_FREE(bat->bat_srv_hash,
1135                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1136         LIBCFS_FREE(bat, sizeof(struct lstcon_batch));
1137 }
1138
1139 static int
1140 lstcon_testrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1141 {
1142         struct lstcon_test *test;
1143         struct lstcon_batch *batch;
1144         struct lstcon_ndlink *ndl;
1145         struct list_head *hash;
1146         struct list_head *head;
1147
1148         test = (struct lstcon_test *)arg;
1149         LASSERT(test);
1150
1151         batch = test->tes_batch;
1152         LASSERT(batch);
1153
1154         if (test->tes_oneside &&
1155             transop == LST_TRANS_TSBSRVADD)
1156                 return 0;
1157
1158         if (nd->nd_state != LST_NODE_ACTIVE)
1159                 return -ENETDOWN;
1160
1161         if (transop == LST_TRANS_TSBCLIADD) {
1162                 hash = batch->bat_cli_hash;
1163                 head = &batch->bat_cli_list;
1164
1165         } else {
1166                 LASSERT(transop == LST_TRANS_TSBSRVADD);
1167
1168                 hash = batch->bat_srv_hash;
1169                 head = &batch->bat_srv_list;
1170         }
1171
1172         LASSERT(nd->nd_id.nid != LNET_NID_ANY);
1173
1174         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1))
1175                 return -ENOMEM;
1176
1177         if (list_empty(&ndl->ndl_link))
1178                 list_add_tail(&ndl->ndl_link, head);
1179
1180         return 1;
1181 }
1182
1183 static int
1184 lstcon_test_nodes_add(struct lstcon_test *test, struct list_head __user *result_up)
1185 {
1186         struct lstcon_rpc_trans *trans;
1187         struct lstcon_group *grp;
1188         int transop;
1189         int rc;
1190
1191         LASSERT(test->tes_src_grp);
1192         LASSERT(test->tes_dst_grp);
1193
1194         transop = LST_TRANS_TSBSRVADD;
1195         grp = test->tes_dst_grp;
1196 again:
1197         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1198                                      &test->tes_trans_list, transop,
1199                                      test, lstcon_testrpc_condition, &trans);
1200         if (rc) {
1201                 CERROR("Can't create transaction: %d\n", rc);
1202                 return rc;
1203         }
1204
1205         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1206
1207         if (lstcon_trans_stat()->trs_rpc_errno ||
1208             lstcon_trans_stat()->trs_fwk_errno) {
1209                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1210
1211                 lstcon_rpc_trans_destroy(trans);
1212                 /* return if any error */
1213                 CDEBUG(D_NET, "Failed to add test %s, RPC error %d, framework error %d\n",
1214                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1215                        lstcon_trans_stat()->trs_rpc_errno,
1216                        lstcon_trans_stat()->trs_fwk_errno);
1217
1218                 return rc;
1219         }
1220
1221         lstcon_rpc_trans_destroy(trans);
1222
1223         if (transop == LST_TRANS_TSBCLIADD)
1224                 return rc;
1225
1226         transop = LST_TRANS_TSBCLIADD;
1227         grp = test->tes_src_grp;
1228         test->tes_cliidx = 0;
1229
1230         /* requests to test clients */
1231         goto again;
1232 }
1233
1234 static int
1235 lstcon_verify_batch(const char *name, struct lstcon_batch **batch)
1236 {
1237         int rc;
1238
1239         rc = lstcon_batch_find(name, batch);
1240         if (rc) {
1241                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1242                 return rc;
1243         }
1244
1245         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1246                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1247                 return -EINVAL;
1248         }
1249
1250         return 0;
1251 }
1252
1253 static int
1254 lstcon_verify_group(const char *name, struct lstcon_group **grp)
1255 {
1256         int rc;
1257         struct lstcon_ndlink    *ndl;
1258
1259         rc = lstcon_group_find(name, grp);
1260         if (rc) {
1261                 CDEBUG(D_NET, "can't find group %s\n", name);
1262                 return rc;
1263         }
1264
1265         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1266                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE)
1267                         return 0;
1268         }
1269
1270         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1271
1272         return -EINVAL;
1273 }
1274
1275 int
1276 lstcon_test_add(char *batch_name, int type, int loop,
1277                 int concur, int dist, int span,
1278                 char *src_name, char *dst_name,
1279                 void *param, int paramlen, int *retp,
1280                 struct list_head __user *result_up)
1281 {
1282         struct lstcon_test *test = NULL;
1283         int rc;
1284         struct lstcon_group *src_grp = NULL;
1285         struct lstcon_group *dst_grp = NULL;
1286         struct lstcon_batch *batch = NULL;
1287
1288         /*
1289          * verify that a batch of the given name exists, and the groups
1290          * that will be part of the batch exist and have at least one
1291          * active node
1292          */
1293         rc = lstcon_verify_batch(batch_name, &batch);
1294         if (rc)
1295                 goto out;
1296
1297         rc = lstcon_verify_group(src_name, &src_grp);
1298         if (rc)
1299                 goto out;
1300
1301         rc = lstcon_verify_group(dst_name, &dst_grp);
1302         if (rc)
1303                 goto out;
1304
1305         if (dst_grp->grp_userland)
1306                 *retp = 1;
1307
1308         LIBCFS_ALLOC(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1309         if (!test) {
1310                 CERROR("Can't allocate test descriptor\n");
1311                 rc = -ENOMEM;
1312
1313                 goto out;
1314         }
1315
1316         test->tes_hdr.tsb_id = batch->bat_hdr.tsb_id;
1317         test->tes_batch = batch;
1318         test->tes_type = type;
1319         test->tes_oneside = 0; /* TODO */
1320         test->tes_loop = loop;
1321         test->tes_concur = concur;
1322         test->tes_stop_onerr = 1; /* TODO */
1323         test->tes_span = span;
1324         test->tes_dist = dist;
1325         test->tes_cliidx = 0; /* just used for creating RPC */
1326         test->tes_src_grp = src_grp;
1327         test->tes_dst_grp = dst_grp;
1328         INIT_LIST_HEAD(&test->tes_trans_list);
1329
1330         if (param) {
1331                 test->tes_paramlen = paramlen;
1332                 memcpy(&test->tes_param[0], param, paramlen);
1333         }
1334
1335         rc = lstcon_test_nodes_add(test, result_up);
1336
1337         if (rc)
1338                 goto out;
1339
1340         if (lstcon_trans_stat()->trs_rpc_errno ||
1341             lstcon_trans_stat()->trs_fwk_errno)
1342                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1343                        batch_name);
1344
1345         /* add to test list anyway, so user can check what's going on */
1346         list_add_tail(&test->tes_link, &batch->bat_test_list);
1347
1348         batch->bat_ntest++;
1349         test->tes_hdr.tsb_index = batch->bat_ntest;
1350
1351         /*  hold groups so nobody can change them */
1352         return rc;
1353 out:
1354         if (test)
1355                 LIBCFS_FREE(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1356
1357         if (dst_grp)
1358                 lstcon_group_decref(dst_grp);
1359
1360         if (src_grp)
1361                 lstcon_group_decref(src_grp);
1362
1363         return rc;
1364 }
1365
1366 static int
1367 lstcon_test_find(struct lstcon_batch *batch, int idx, struct lstcon_test **testpp)
1368 {
1369         struct lstcon_test *test;
1370
1371         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1372                 if (idx == test->tes_hdr.tsb_index) {
1373                         *testpp = test;
1374                         return 0;
1375                 }
1376         }
1377
1378         return -ENOENT;
1379 }
1380
1381 static int
1382 lstcon_tsbrpc_readent(int transop, struct srpc_msg *msg,
1383                       lstcon_rpc_ent_t __user *ent_up)
1384 {
1385         struct srpc_batch_reply *rep = &msg->msg_body.bat_reply;
1386
1387         LASSERT(transop == LST_TRANS_TSBCLIQRY ||
1388                 transop == LST_TRANS_TSBSRVQRY);
1389
1390         /* positive errno, framework error code */
1391         if (copy_to_user(&ent_up->rpe_priv[0], &rep->bar_active,
1392                          sizeof(rep->bar_active)))
1393                 return -EFAULT;
1394
1395         return 0;
1396 }
1397
1398 int
1399 lstcon_test_batch_query(char *name, int testidx, int client,
1400                         int timeout, struct list_head __user *result_up)
1401 {
1402         struct lstcon_rpc_trans *trans;
1403         struct list_head *translist;
1404         struct list_head *ndlist;
1405         struct lstcon_tsb_hdr *hdr;
1406         struct lstcon_batch *batch;
1407         struct lstcon_test *test = NULL;
1408         int transop;
1409         int rc;
1410
1411         rc = lstcon_batch_find(name, &batch);
1412         if (rc) {
1413                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1414                 return rc;
1415         }
1416
1417         if (!testidx) {
1418                 translist = &batch->bat_trans_list;
1419                 ndlist = &batch->bat_cli_list;
1420                 hdr = &batch->bat_hdr;
1421         } else {
1422                 /* query specified test only */
1423                 rc = lstcon_test_find(batch, testidx, &test);
1424                 if (rc) {
1425                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1426                         return rc;
1427                 }
1428
1429                 translist = &test->tes_trans_list;
1430                 ndlist = &test->tes_src_grp->grp_ndl_list;
1431                 hdr = &test->tes_hdr;
1432         }
1433
1434         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1435
1436         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1437                                      lstcon_batrpc_condition, &trans);
1438         if (rc) {
1439                 CERROR("Can't create transaction: %d\n", rc);
1440                 return rc;
1441         }
1442
1443         lstcon_rpc_trans_postwait(trans, timeout);
1444
1445         /* query a batch, not a test */
1446         if (!testidx &&
1447             !lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) &&
1448             !lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0)) {
1449                 /* all RPCs finished, and no active test */
1450                 batch->bat_state = LST_BATCH_IDLE;
1451         }
1452
1453         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1454                                           lstcon_tsbrpc_readent);
1455         lstcon_rpc_trans_destroy(trans);
1456
1457         return rc;
1458 }
1459
1460 static int
1461 lstcon_statrpc_readent(int transop, struct srpc_msg *msg,
1462                        lstcon_rpc_ent_t __user *ent_up)
1463 {
1464         struct srpc_stat_reply *rep = &msg->msg_body.stat_reply;
1465         sfw_counters_t __user *sfwk_stat;
1466         srpc_counters_t __user *srpc_stat;
1467         lnet_counters_t __user *lnet_stat;
1468
1469         if (rep->str_status)
1470                 return 0;
1471
1472         sfwk_stat = (sfw_counters_t __user *)&ent_up->rpe_payload[0];
1473         srpc_stat = (srpc_counters_t __user *)(sfwk_stat + 1);
1474         lnet_stat = (lnet_counters_t __user *)(srpc_stat + 1);
1475
1476         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1477             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1478             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1479                 return -EFAULT;
1480
1481         return 0;
1482 }
1483
1484 static int
1485 lstcon_ndlist_stat(struct list_head *ndlist,
1486                    int timeout, struct list_head __user *result_up)
1487 {
1488         struct list_head head;
1489         struct lstcon_rpc_trans *trans;
1490         int rc;
1491
1492         INIT_LIST_HEAD(&head);
1493
1494         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1495                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1496         if (rc) {
1497                 CERROR("Can't create transaction: %d\n", rc);
1498                 return rc;
1499         }
1500
1501         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1502
1503         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1504                                           lstcon_statrpc_readent);
1505         lstcon_rpc_trans_destroy(trans);
1506
1507         return rc;
1508 }
1509
1510 int
1511 lstcon_group_stat(char *grp_name, int timeout,
1512                   struct list_head __user *result_up)
1513 {
1514         struct lstcon_group *grp;
1515         int rc;
1516
1517         rc = lstcon_group_find(grp_name, &grp);
1518         if (rc) {
1519                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1520                 return rc;
1521         }
1522
1523         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1524
1525         lstcon_group_decref(grp);
1526
1527         return rc;
1528 }
1529
1530 int
1531 lstcon_nodes_stat(int count, lnet_process_id_t __user *ids_up,
1532                   int timeout, struct list_head __user *result_up)
1533 {
1534         struct lstcon_ndlink    *ndl;
1535         struct lstcon_group *tmp;
1536         lnet_process_id_t id;
1537         int i;
1538         int rc;
1539
1540         rc = lstcon_group_alloc(NULL, &tmp);
1541         if (rc) {
1542                 CERROR("Out of memory\n");
1543                 return -ENOMEM;
1544         }
1545
1546         for (i = 0 ; i < count; i++) {
1547                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1548                         rc = -EFAULT;
1549                         break;
1550                 }
1551
1552                 /* add to tmp group */
1553                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1554                 if (rc) {
1555                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1556                                "Failed to find or create %s: %d\n",
1557                                libcfs_id2str(id), rc);
1558                         break;
1559                 }
1560         }
1561
1562         if (rc) {
1563                 lstcon_group_decref(tmp);
1564                 return rc;
1565         }
1566
1567         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1568
1569         lstcon_group_decref(tmp);
1570
1571         return rc;
1572 }
1573
1574 static int
1575 lstcon_debug_ndlist(struct list_head *ndlist,
1576                     struct list_head *translist,
1577                     int timeout, struct list_head __user *result_up)
1578 {
1579         struct lstcon_rpc_trans *trans;
1580         int rc;
1581
1582         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1583                                      NULL, lstcon_sesrpc_condition, &trans);
1584         if (rc) {
1585                 CERROR("Can't create transaction: %d\n", rc);
1586                 return rc;
1587         }
1588
1589         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1590
1591         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1592                                           lstcon_sesrpc_readent);
1593         lstcon_rpc_trans_destroy(trans);
1594
1595         return rc;
1596 }
1597
1598 int
1599 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1600 {
1601         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1602                                    NULL, timeout, result_up);
1603 }
1604
1605 int
1606 lstcon_batch_debug(int timeout, char *name,
1607                    int client, struct list_head __user *result_up)
1608 {
1609         struct lstcon_batch *bat;
1610         int rc;
1611
1612         rc = lstcon_batch_find(name, &bat);
1613         if (rc)
1614                 return -ENOENT;
1615
1616         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1617                                           &bat->bat_srv_list,
1618                                  NULL, timeout, result_up);
1619
1620         return rc;
1621 }
1622
1623 int
1624 lstcon_group_debug(int timeout, char *name,
1625                    struct list_head __user *result_up)
1626 {
1627         struct lstcon_group *grp;
1628         int rc;
1629
1630         rc = lstcon_group_find(name, &grp);
1631         if (rc)
1632                 return -ENOENT;
1633
1634         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1635                                  timeout, result_up);
1636         lstcon_group_decref(grp);
1637
1638         return rc;
1639 }
1640
1641 int
1642 lstcon_nodes_debug(int timeout,
1643                    int count, lnet_process_id_t __user *ids_up,
1644                    struct list_head __user *result_up)
1645 {
1646         lnet_process_id_t id;
1647         struct lstcon_ndlink *ndl;
1648         struct lstcon_group *grp;
1649         int i;
1650         int rc;
1651
1652         rc = lstcon_group_alloc(NULL, &grp);
1653         if (rc) {
1654                 CDEBUG(D_NET, "Out of memory\n");
1655                 return rc;
1656         }
1657
1658         for (i = 0; i < count; i++) {
1659                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1660                         rc = -EFAULT;
1661                         break;
1662                 }
1663
1664                 /* node is added to tmp group */
1665                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1666                 if (rc) {
1667                         CERROR("Can't create node link\n");
1668                         break;
1669                 }
1670         }
1671
1672         if (rc) {
1673                 lstcon_group_decref(grp);
1674                 return rc;
1675         }
1676
1677         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1678                                  timeout, result_up);
1679
1680         lstcon_group_decref(grp);
1681
1682         return rc;
1683 }
1684
1685 int
1686 lstcon_session_match(lst_sid_t sid)
1687 {
1688         return (console_session.ses_id.ses_nid == sid.ses_nid &&
1689                 console_session.ses_id.ses_stamp == sid.ses_stamp) ? 1 : 0;
1690 }
1691
1692 static void
1693 lstcon_new_session_id(lst_sid_t *sid)
1694 {
1695         lnet_process_id_t id;
1696
1697         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1698
1699         LNetGetId(1, &id);
1700         sid->ses_nid = id.nid;
1701         sid->ses_stamp = cfs_time_current();
1702 }
1703
1704 int
1705 lstcon_session_new(char *name, int key, unsigned feats,
1706                    int timeout, int force, lst_sid_t __user *sid_up)
1707 {
1708         int rc = 0;
1709         int i;
1710
1711         if (console_session.ses_state != LST_SESSION_NONE) {
1712                 /* session exists */
1713                 if (!force) {
1714                         CNETERR("Session %s already exists\n",
1715                                 console_session.ses_name);
1716                         return -EEXIST;
1717                 }
1718
1719                 rc = lstcon_session_end();
1720
1721                 /* lstcon_session_end() only return local error */
1722                 if (rc)
1723                         return rc;
1724         }
1725
1726         if (feats & ~LST_FEATS_MASK) {
1727                 CNETERR("Unknown session features %x\n",
1728                         (feats & ~LST_FEATS_MASK));
1729                 return -EINVAL;
1730         }
1731
1732         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1733                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1734
1735         lstcon_new_session_id(&console_session.ses_id);
1736
1737         console_session.ses_key = key;
1738         console_session.ses_state = LST_SESSION_ACTIVE;
1739         console_session.ses_force = !!force;
1740         console_session.ses_features = feats;
1741         console_session.ses_feats_updated = 0;
1742         console_session.ses_timeout = (timeout <= 0) ?
1743                                       LST_CONSOLE_TIMEOUT : timeout;
1744
1745         if (strlen(name) > sizeof(console_session.ses_name) - 1)
1746                 return -E2BIG;
1747         strlcpy(console_session.ses_name, name,
1748                 sizeof(console_session.ses_name));
1749
1750         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1751         if (rc)
1752                 return rc;
1753
1754         rc = lstcon_rpc_pinger_start();
1755         if (rc) {
1756                 struct lstcon_batch *bat = NULL;
1757
1758                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1759                 lstcon_batch_destroy(bat);
1760
1761                 return rc;
1762         }
1763
1764         if (!copy_to_user(sid_up, &console_session.ses_id,
1765                           sizeof(lst_sid_t)))
1766                 return rc;
1767
1768         lstcon_session_end();
1769
1770         return -EFAULT;
1771 }
1772
1773 int
1774 lstcon_session_info(lst_sid_t __user *sid_up, int __user *key_up,
1775                     unsigned __user *featp,
1776                     lstcon_ndlist_ent_t __user *ndinfo_up,
1777                     char __user *name_up, int len)
1778 {
1779         lstcon_ndlist_ent_t *entp;
1780         struct lstcon_ndlink *ndl;
1781         int rc = 0;
1782
1783         if (console_session.ses_state != LST_SESSION_ACTIVE)
1784                 return -ESRCH;
1785
1786         LIBCFS_ALLOC(entp, sizeof(*entp));
1787         if (!entp)
1788                 return -ENOMEM;
1789
1790         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1791                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1792
1793         if (copy_to_user(sid_up, &console_session.ses_id,
1794                          sizeof(lst_sid_t)) ||
1795             copy_to_user(key_up, &console_session.ses_key,
1796                          sizeof(*key_up)) ||
1797             copy_to_user(featp, &console_session.ses_features,
1798                          sizeof(*featp)) ||
1799             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1800             copy_to_user(name_up, console_session.ses_name, len))
1801                 rc = -EFAULT;
1802
1803         LIBCFS_FREE(entp, sizeof(*entp));
1804
1805         return rc;
1806 }
1807
1808 int
1809 lstcon_session_end(void)
1810 {
1811         struct lstcon_rpc_trans *trans;
1812         struct lstcon_group *grp;
1813         struct lstcon_batch *bat;
1814         int rc = 0;
1815
1816         LASSERT(console_session.ses_state == LST_SESSION_ACTIVE);
1817
1818         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1819                                      NULL, LST_TRANS_SESEND, NULL,
1820                                      lstcon_sesrpc_condition, &trans);
1821         if (rc) {
1822                 CERROR("Can't create transaction: %d\n", rc);
1823                 return rc;
1824         }
1825
1826         console_session.ses_shutdown = 1;
1827
1828         lstcon_rpc_pinger_stop();
1829
1830         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1831
1832         lstcon_rpc_trans_destroy(trans);
1833         /* User can do nothing even rpc failed, so go on */
1834
1835         /* waiting for orphan rpcs to die */
1836         lstcon_rpc_cleanup_wait();
1837
1838         console_session.ses_id = LST_INVALID_SID;
1839         console_session.ses_state = LST_SESSION_NONE;
1840         console_session.ses_key = 0;
1841         console_session.ses_force = 0;
1842         console_session.ses_feats_updated = 0;
1843
1844         /* destroy all batches */
1845         while (!list_empty(&console_session.ses_bat_list)) {
1846                 bat = list_entry(console_session.ses_bat_list.next,
1847                                  struct lstcon_batch, bat_link);
1848
1849                 lstcon_batch_destroy(bat);
1850         }
1851
1852         /* destroy all groups */
1853         while (!list_empty(&console_session.ses_grp_list)) {
1854                 grp = list_entry(console_session.ses_grp_list.next,
1855                                  struct lstcon_group, grp_link);
1856                 LASSERT(grp->grp_ref == 1);
1857
1858                 lstcon_group_decref(grp);
1859         }
1860
1861         /* all nodes should be released */
1862         LASSERT(list_empty(&console_session.ses_ndl_list));
1863
1864         console_session.ses_shutdown = 0;
1865         console_session.ses_expired = 0;
1866
1867         return rc;
1868 }
1869
1870 int
1871 lstcon_session_feats_check(unsigned feats)
1872 {
1873         int rc = 0;
1874
1875         if (feats & ~LST_FEATS_MASK) {
1876                 CERROR("Can't support these features: %x\n",
1877                        (feats & ~LST_FEATS_MASK));
1878                 return -EPROTO;
1879         }
1880
1881         spin_lock(&console_session.ses_rpc_lock);
1882
1883         if (!console_session.ses_feats_updated) {
1884                 console_session.ses_feats_updated = 1;
1885                 console_session.ses_features = feats;
1886         }
1887
1888         if (console_session.ses_features != feats)
1889                 rc = -EPROTO;
1890
1891         spin_unlock(&console_session.ses_rpc_lock);
1892
1893         if (rc) {
1894                 CERROR("remote features %x do not match with session features %x of console\n",
1895                        feats, console_session.ses_features);
1896         }
1897
1898         return rc;
1899 }
1900
1901 static int
1902 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1903 {
1904         struct srpc_msg *rep    = &rpc->srpc_replymsg;
1905         struct srpc_msg *req    = &rpc->srpc_reqstbuf->buf_msg;
1906         struct srpc_join_reqst *jreq = &req->msg_body.join_reqst;
1907         struct srpc_join_reply *jrep = &rep->msg_body.join_reply;
1908         struct lstcon_group *grp = NULL;
1909         struct lstcon_ndlink *ndl;
1910         int rc = 0;
1911
1912         sfw_unpack_message(req);
1913
1914         mutex_lock(&console_session.ses_mutex);
1915
1916         jrep->join_sid = console_session.ses_id;
1917
1918         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1919                 jrep->join_status = ESRCH;
1920                 goto out;
1921         }
1922
1923         if (lstcon_session_feats_check(req->msg_ses_feats)) {
1924                 jrep->join_status = EPROTO;
1925                 goto out;
1926         }
1927
1928         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1929             !lstcon_session_match(jreq->join_sid)) {
1930                 jrep->join_status = EBUSY;
1931                 goto out;
1932         }
1933
1934         if (lstcon_group_find(jreq->join_group, &grp)) {
1935                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1936                 if (rc) {
1937                         CERROR("Out of memory\n");
1938                         goto out;
1939                 }
1940
1941                 list_add_tail(&grp->grp_link,
1942                               &console_session.ses_grp_list);
1943                 lstcon_group_addref(grp);
1944         }
1945
1946         if (grp->grp_ref > 2) {
1947                 /* Group in using */
1948                 jrep->join_status = EBUSY;
1949                 goto out;
1950         }
1951
1952         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1953         if (!rc) {
1954                 jrep->join_status = EEXIST;
1955                 goto out;
1956         }
1957
1958         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1959         if (rc) {
1960                 CERROR("Out of memory\n");
1961                 goto out;
1962         }
1963
1964         ndl->ndl_node->nd_state = LST_NODE_ACTIVE;
1965         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1966
1967         if (!grp->grp_userland)
1968                 grp->grp_userland = 1;
1969
1970         strlcpy(jrep->join_session, console_session.ses_name,
1971                 sizeof(jrep->join_session));
1972         jrep->join_timeout = console_session.ses_timeout;
1973         jrep->join_status = 0;
1974
1975 out:
1976         rep->msg_ses_feats = console_session.ses_features;
1977         if (grp)
1978                 lstcon_group_decref(grp);
1979
1980         mutex_unlock(&console_session.ses_mutex);
1981
1982         return rc;
1983 }
1984
1985 static struct srpc_service lstcon_acceptor_service;
1986
1987 static void lstcon_init_acceptor_service(void)
1988 {
1989         /* initialize selftest console acceptor service table */
1990         lstcon_acceptor_service.sv_name = "join session";
1991         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
1992         lstcon_acceptor_service.sv_id = SRPC_SERVICE_JOIN;
1993         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1994 }
1995
1996 static DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1997
1998 /* initialize console */
1999 int
2000 lstcon_console_init(void)
2001 {
2002         int i;
2003         int rc;
2004
2005         memset(&console_session, 0, sizeof(struct lstcon_session));
2006
2007         console_session.ses_id = LST_INVALID_SID;
2008         console_session.ses_state = LST_SESSION_NONE;
2009         console_session.ses_timeout = 0;
2010         console_session.ses_force = 0;
2011         console_session.ses_expired = 0;
2012         console_session.ses_feats_updated = 0;
2013         console_session.ses_features = LST_FEATS_MASK;
2014         console_session.ses_laststamp = ktime_get_real_seconds();
2015
2016         mutex_init(&console_session.ses_mutex);
2017
2018         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2019         INIT_LIST_HEAD(&console_session.ses_grp_list);
2020         INIT_LIST_HEAD(&console_session.ses_bat_list);
2021         INIT_LIST_HEAD(&console_session.ses_trans_list);
2022
2023         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2024                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2025         if (!console_session.ses_ndl_hash)
2026                 return -ENOMEM;
2027
2028         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2029                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2030
2031         /* initialize acceptor service table */
2032         lstcon_init_acceptor_service();
2033
2034         rc = srpc_add_service(&lstcon_acceptor_service);
2035         LASSERT(rc != -EBUSY);
2036         if (rc) {
2037                 LIBCFS_FREE(console_session.ses_ndl_hash,
2038                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2039                 return rc;
2040         }
2041
2042         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2043                                       lstcon_acceptor_service.sv_wi_total);
2044         if (rc) {
2045                 rc = -ENOMEM;
2046                 goto out;
2047         }
2048
2049         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2050
2051         if (!rc) {
2052                 lstcon_rpc_module_init();
2053                 return 0;
2054         }
2055
2056 out:
2057         srpc_shutdown_service(&lstcon_acceptor_service);
2058         srpc_remove_service(&lstcon_acceptor_service);
2059
2060         LIBCFS_FREE(console_session.ses_ndl_hash,
2061                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2062
2063         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2064
2065         return rc;
2066 }
2067
2068 int
2069 lstcon_console_fini(void)
2070 {
2071         int i;
2072
2073         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2074
2075         mutex_lock(&console_session.ses_mutex);
2076
2077         srpc_shutdown_service(&lstcon_acceptor_service);
2078         srpc_remove_service(&lstcon_acceptor_service);
2079
2080         if (console_session.ses_state != LST_SESSION_NONE)
2081                 lstcon_session_end();
2082
2083         lstcon_rpc_module_fini();
2084
2085         mutex_unlock(&console_session.ses_mutex);
2086
2087         LASSERT(list_empty(&console_session.ses_ndl_list));
2088         LASSERT(list_empty(&console_session.ses_grp_list));
2089         LASSERT(list_empty(&console_session.ses_bat_list));
2090         LASSERT(list_empty(&console_session.ses_trans_list));
2091
2092         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2093                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2094
2095         LIBCFS_FREE(console_session.ses_ndl_hash,
2096                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2097
2098         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2099
2100         return 0;
2101 }