drivers/staging/lustre/lustre/lov/lov_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2015, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * Implementation of cl_device and cl_device_type for LOV layer.
  33  *
  34  *   Author: Nikita Danilov <nikita.danilov@sun.com>
  35  */
  36
  37 #define DEBUG_SUBSYSTEM S_LOV
  38
  39 /* class_name2obd() */
  40 #include "../include/obd_class.h"
  41
  42 #include "lov_cl_internal.h"
  43 #include "lov_internal.h"
  44
  45 struct kmem_cache *lov_lock_kmem;
  46 struct kmem_cache *lov_object_kmem;
  47 struct kmem_cache *lov_thread_kmem;
  48 struct kmem_cache *lov_session_kmem;
  49 struct kmem_cache *lov_req_kmem;
  50
  51 struct kmem_cache *lovsub_lock_kmem;
  52 struct kmem_cache *lovsub_object_kmem;
  53 struct kmem_cache *lovsub_req_kmem;
  54
  55 struct kmem_cache *lov_lock_link_kmem;
  56
  57 /** Lock class of lov_device::ld_mutex. */
  58 static struct lock_class_key cl_lov_device_mutex_class;
  59
  60 struct lu_kmem_descr lov_caches[] = {
  61         {
  62                 .ckd_cache = &lov_lock_kmem,
  63                 .ckd_name  = "lov_lock_kmem",
  64                 .ckd_size  = sizeof(struct lov_lock)
  65         },
  66         {
  67                 .ckd_cache = &lov_object_kmem,
  68                 .ckd_name  = "lov_object_kmem",
  69                 .ckd_size  = sizeof(struct lov_object)
  70         },
  71         {
  72                 .ckd_cache = &lov_thread_kmem,
  73                 .ckd_name  = "lov_thread_kmem",
  74                 .ckd_size  = sizeof(struct lov_thread_info)
  75         },
  76         {
  77                 .ckd_cache = &lov_session_kmem,
  78                 .ckd_name  = "lov_session_kmem",
  79                 .ckd_size  = sizeof(struct lov_session)
  80         },
  81         {
  82                 .ckd_cache = &lov_req_kmem,
  83                 .ckd_name  = "lov_req_kmem",
  84                 .ckd_size  = sizeof(struct lov_req)
  85         },
  86         {
  87                 .ckd_cache = &lovsub_lock_kmem,
  88                 .ckd_name  = "lovsub_lock_kmem",
  89                 .ckd_size  = sizeof(struct lovsub_lock)
  90         },
  91         {
  92                 .ckd_cache = &lovsub_object_kmem,
  93                 .ckd_name  = "lovsub_object_kmem",
  94                 .ckd_size  = sizeof(struct lovsub_object)
  95         },
  96         {
  97                 .ckd_cache = &lovsub_req_kmem,
  98                 .ckd_name  = "lovsub_req_kmem",
  99                 .ckd_size  = sizeof(struct lovsub_req)
 100         },
 101         {
 102                 .ckd_cache = &lov_lock_link_kmem,
 103                 .ckd_name  = "lov_lock_link_kmem",
 104                 .ckd_size  = sizeof(struct lov_lock_link)
 105         },
 106         {
 107                 .ckd_cache = NULL
 108         }
 109 };
 110
 111 /*****************************************************************************
 112  *
 113  * Lov transfer operations.
 114  *
 115  */
 116
 117 static void lov_req_completion(const struct lu_env *env,
 118                                const struct cl_req_slice *slice, int ioret)
 119 {
 120         struct lov_req *lr;
 121
 122         lr = cl2lov_req(slice);
 123         kmem_cache_free(lov_req_kmem, lr);
 124 }
 125
 126 static const struct cl_req_operations lov_req_ops = {
 127         .cro_completion = lov_req_completion
 128 };
 129
 130 /*****************************************************************************
 131  *
 132  * Lov device and device type functions.
 133  *
 134  */
 135
 136 static void *lov_key_init(const struct lu_context *ctx,
 137                           struct lu_context_key *key)
 138 {
 139         struct lov_thread_info *info;
 140
 141         info = kmem_cache_zalloc(lov_thread_kmem, GFP_NOFS);
 142         if (!info)
 143                 info = ERR_PTR(-ENOMEM);
 144         return info;
 145 }
 146
 147 static void lov_key_fini(const struct lu_context *ctx,
 148                          struct lu_context_key *key, void *data)
 149 {
 150         struct lov_thread_info *info = data;
 151
 152         kmem_cache_free(lov_thread_kmem, info);
 153 }
 154
 155 struct lu_context_key lov_key = {
 156         .lct_tags = LCT_CL_THREAD,
 157         .lct_init = lov_key_init,
 158         .lct_fini = lov_key_fini
 159 };
 160
 161 static void *lov_session_key_init(const struct lu_context *ctx,
 162                                   struct lu_context_key *key)
 163 {
 164         struct lov_session *info;
 165
 166         info = kmem_cache_zalloc(lov_session_kmem, GFP_NOFS);
 167         if (!info)
 168                 info = ERR_PTR(-ENOMEM);
 169         return info;
 170 }
 171
 172 static void lov_session_key_fini(const struct lu_context *ctx,
 173                                  struct lu_context_key *key, void *data)
 174 {
 175         struct lov_session *info = data;
 176
 177         kmem_cache_free(lov_session_kmem, info);
 178 }
 179
 180 struct lu_context_key lov_session_key = {
 181         .lct_tags = LCT_SESSION,
 182         .lct_init = lov_session_key_init,
 183         .lct_fini = lov_session_key_fini
 184 };
 185
 186 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
 187 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
 188
 189 static struct lu_device *lov_device_fini(const struct lu_env *env,
 190                                          struct lu_device *d)
 191 {
 192         int i;
 193         struct lov_device *ld = lu2lov_dev(d);
 194
 195         LASSERT(ld->ld_lov);
 196         if (!ld->ld_target)
 197                 return NULL;
 198
 199         lov_foreach_target(ld, i) {
 200                 struct lovsub_device *lsd;
 201
 202                 lsd = ld->ld_target[i];
 203                 if (lsd) {
 204                         cl_stack_fini(env, lovsub2cl_dev(lsd));
 205                         ld->ld_target[i] = NULL;
 206                 }
 207         }
 208         return NULL;
 209 }
 210
 211 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 212                            const char *name, struct lu_device *next)
 213 {
 214         struct lov_device *ld = lu2lov_dev(d);
 215         int i;
 216         int rc = 0;
 217
 218         LASSERT(d->ld_site);
 219         if (!ld->ld_target)
 220                 return rc;
 221
 222         lov_foreach_target(ld, i) {
 223                 struct lovsub_device *lsd;
 224                 struct cl_device     *cl;
 225                 struct lov_tgt_desc  *desc;
 226
 227                 desc = ld->ld_lov->lov_tgts[i];
 228                 if (!desc)
 229                         continue;
 230
 231                 cl = cl_type_setup(env, d->ld_site, &lovsub_device_type,
 232                                    desc->ltd_obd->obd_lu_dev);
 233                 if (IS_ERR(cl)) {
 234                         rc = PTR_ERR(cl);
 235                         break;
 236                 }
 237                 lsd = cl2lovsub_dev(cl);
 238                 lsd->acid_idx = i;
 239                 lsd->acid_super = ld;
 240                 ld->ld_target[i] = lsd;
 241         }
 242
 243         if (rc)
 244                 lov_device_fini(env, d);
 245         else
 246                 ld->ld_flags |= LOV_DEV_INITIALIZED;
 247
 248         return rc;
 249 }
 250
 251 static int lov_req_init(const struct lu_env *env, struct cl_device *dev,
 252                         struct cl_req *req)
 253 {
 254         struct lov_req *lr;
 255         int result;
 256
 257         lr = kmem_cache_zalloc(lov_req_kmem, GFP_NOFS);
 258         if (lr) {
 259                 cl_req_slice_add(req, &lr->lr_cl, dev, &lov_req_ops);
 260                 result = 0;
 261         } else {
 262                 result = -ENOMEM;
 263         }
 264         return result;
 265 }
 266
 267 static const struct cl_device_operations lov_cl_ops = {
 268         .cdo_req_init = lov_req_init
 269 };
 270
 271 static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
 272 {
 273         int i;
 274
 275         for (i = 0; i < nr; ++i) {
 276                 struct lov_device_emerg *em;
 277
 278                 em = emrg[i];
 279                 if (em) {
 280                         LASSERT(em->emrg_page_list.pl_nr == 0);
 281                         if (em->emrg_env)
 282                                 cl_env_put(em->emrg_env, &em->emrg_refcheck);
 283                         kfree(em);
 284                 }
 285         }
 286         kfree(emrg);
 287 }
 288
 289 static struct lu_device *lov_device_free(const struct lu_env *env,
 290                                          struct lu_device *d)
 291 {
 292         struct lov_device *ld = lu2lov_dev(d);
 293         const int         nr = ld->ld_target_nr;
 294
 295         cl_device_fini(lu2cl_dev(d));
 296         kfree(ld->ld_target);
 297         if (ld->ld_emrg)
 298                 lov_emerg_free(ld->ld_emrg, nr);
 299         kfree(ld);
 300         return NULL;
 301 }
 302
 303 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
 304                               __u32 index)
 305 {
 306         struct lov_device *ld = lu2lov_dev(dev);
 307
 308         if (ld->ld_target[index]) {
 309                 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
 310                 ld->ld_target[index] = NULL;
 311         }
 312 }
 313
 314 static struct lov_device_emerg **lov_emerg_alloc(int nr)
 315 {
 316         struct lov_device_emerg **emerg;
 317         int i;
 318         int result;
 319
 320         emerg = kcalloc(nr, sizeof(emerg[0]), GFP_NOFS);
 321         if (!emerg)
 322                 return ERR_PTR(-ENOMEM);
 323         for (result = i = 0; i < nr && result == 0; i++) {
 324                 struct lov_device_emerg *em;
 325
 326                 em = kzalloc(sizeof(*em), GFP_NOFS);
 327                 if (em) {
 328                         emerg[i] = em;
 329                         cl_page_list_init(&em->emrg_page_list);
 330                         em->emrg_env = cl_env_alloc(&em->emrg_refcheck,
 331                                                     LCT_REMEMBER | LCT_NOREF);
 332                         if (!IS_ERR(em->emrg_env)) {
 333                                 em->emrg_env->le_ctx.lc_cookie = 0x2;
 334                         } else {
 335                                 result = PTR_ERR(em->emrg_env);
 336                                 em->emrg_env = NULL;
 337                         }
 338                 } else {
 339                         result = -ENOMEM;
 340                 }
 341         }
 342         if (result != 0) {
 343                 lov_emerg_free(emerg, nr);
 344                 emerg = ERR_PTR(result);
 345         }
 346         return emerg;
 347 }
 348
 349 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
 350 {
 351         int   result;
 352         __u32 tgt_size;
 353         __u32 sub_size;
 354
 355         result = 0;
 356         tgt_size = dev->ld_lov->lov_tgt_size;
 357         sub_size = dev->ld_target_nr;
 358         if (sub_size < tgt_size) {
 359                 struct lovsub_device    **newd;
 360                 struct lov_device_emerg **emerg;
 361                 const size_t          sz   = sizeof(newd[0]);
 362
 363                 emerg = lov_emerg_alloc(tgt_size);
 364                 if (IS_ERR(emerg))
 365                         return PTR_ERR(emerg);
 366
 367                 newd = kcalloc(tgt_size, sz, GFP_NOFS);
 368                 if (newd) {
 369                         mutex_lock(&dev->ld_mutex);
 370                         if (sub_size > 0) {
 371                                 memcpy(newd, dev->ld_target, sub_size * sz);
 372                                 kfree(dev->ld_target);
 373                         }
 374                         dev->ld_target    = newd;
 375                         dev->ld_target_nr = tgt_size;
 376
 377                         if (dev->ld_emrg)
 378                                 lov_emerg_free(dev->ld_emrg, sub_size);
 379                         dev->ld_emrg = emerg;
 380                         mutex_unlock(&dev->ld_mutex);
 381                 } else {
 382                         lov_emerg_free(emerg, tgt_size);
 383                         result = -ENOMEM;
 384                 }
 385         }
 386         return result;
 387 }
 388
 389 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
 390                              __u32 index)
 391 {
 392         struct obd_device    *obd = dev->ld_obd;
 393         struct lov_device    *ld  = lu2lov_dev(dev);
 394         struct lov_tgt_desc  *tgt;
 395         struct lovsub_device *lsd;
 396         struct cl_device     *cl;
 397         int rc;
 398
 399         obd_getref(obd);
 400
 401         tgt = obd->u.lov.lov_tgts[index];
 402
 403         if (!tgt->ltd_obd->obd_set_up) {
 404                 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
 405                 return -EINVAL;
 406         }
 407
 408         rc = lov_expand_targets(env, ld);
 409         if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
 410                 LASSERT(dev->ld_site);
 411
 412                 cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type,
 413                                    tgt->ltd_obd->obd_lu_dev);
 414                 if (!IS_ERR(cl)) {
 415                         lsd = cl2lovsub_dev(cl);
 416                         lsd->acid_idx = index;
 417                         lsd->acid_super = ld;
 418                         ld->ld_target[index] = lsd;
 419                 } else {
 420                         CERROR("add failed (%d), deleting %s\n", rc,
 421                                obd_uuid2str(&tgt->ltd_uuid));
 422                         lov_cl_del_target(env, dev, index);
 423                         rc = PTR_ERR(cl);
 424                 }
 425         }
 426         obd_putref(obd);
 427         return rc;
 428 }
 429
 430 static int lov_process_config(const struct lu_env *env,
 431                               struct lu_device *d, struct lustre_cfg *cfg)
 432 {
 433         struct obd_device *obd = d->ld_obd;
 434         int cmd;
 435         int rc;
 436         int gen;
 437         __u32 index;
 438
 439         obd_getref(obd);
 440
 441         cmd = cfg->lcfg_command;
 442         rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
 443         if (rc == 0) {
 444                 switch (cmd) {
 445                 case LCFG_LOV_ADD_OBD:
 446                 case LCFG_LOV_ADD_INA:
 447                         rc = lov_cl_add_target(env, d, index);
 448                         if (rc != 0)
 449                                 lov_del_target(d->ld_obd, index, NULL, 0);
 450                         break;
 451                 case LCFG_LOV_DEL_OBD:
 452                         lov_cl_del_target(env, d, index);
 453                         break;
 454                 }
 455         }
 456         obd_putref(obd);
 457         return rc;
 458 }
 459
 460 static const struct lu_device_operations lov_lu_ops = {
 461         .ldo_object_alloc      = lov_object_alloc,
 462         .ldo_process_config    = lov_process_config,
 463 };
 464
 465 static struct lu_device *lov_device_alloc(const struct lu_env *env,
 466                                           struct lu_device_type *t,
 467                                           struct lustre_cfg *cfg)
 468 {
 469         struct lu_device *d;
 470         struct lov_device *ld;
 471         struct obd_device *obd;
 472         int rc;
 473
 474         ld = kzalloc(sizeof(*ld), GFP_NOFS);
 475         if (!ld)
 476                 return ERR_PTR(-ENOMEM);
 477
 478         cl_device_init(&ld->ld_cl, t);
 479         d = lov2lu_dev(ld);
 480         d->ld_ops       = &lov_lu_ops;
 481         ld->ld_cl.cd_ops = &lov_cl_ops;
 482
 483         mutex_init(&ld->ld_mutex);
 484         lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
 485
 486         /* setup the LOV OBD */
 487         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 488         LASSERT(obd);
 489         rc = lov_setup(obd, cfg);
 490         if (rc) {
 491                 lov_device_free(env, d);
 492                 return ERR_PTR(rc);
 493         }
 494
 495         ld->ld_lov = &obd->u.lov;
 496         return d;
 497 }
 498
 499 static const struct lu_device_type_operations lov_device_type_ops = {
 500         .ldto_init = lov_type_init,
 501         .ldto_fini = lov_type_fini,
 502
 503         .ldto_start = lov_type_start,
 504         .ldto_stop  = lov_type_stop,
 505
 506         .ldto_device_alloc = lov_device_alloc,
 507         .ldto_device_free  = lov_device_free,
 508
 509         .ldto_device_init    = lov_device_init,
 510         .ldto_device_fini    = lov_device_fini
 511 };
 512
 513 struct lu_device_type lov_device_type = {
 514         .ldt_tags     = LU_DEVICE_CL,
 515         .ldt_name     = LUSTRE_LOV_NAME,
 516         .ldt_ops      = &lov_device_type_ops,
 517         .ldt_ctx_tags = LCT_CL_THREAD
 518 };
 519
 520 /** @} lov */