kernel/trace/trace_probe.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common code for probe-based Dynamic events.
   4  *
   5  * This code was copied from kernel/trace/trace_kprobe.c written by
   6  * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
   7  *
   8  * Updates to make this generic:
   9  * Copyright (C) IBM Corporation, 2010-2011
  10  * Author:     Srikar Dronamraju
  11  */
  12 #define pr_fmt(fmt)     "trace_probe: " fmt
  13
  14 #include <linux/bpf.h>
  15 #include "trace_btf.h"
  16
  17 #include "trace_probe.h"
  18
  19 #undef C
  20 #define C(a, b)         b
  21
  22 static const char *trace_probe_err_text[] = { ERRORS };
  23
  24 static const char *reserved_field_names[] = {
  25         "common_type",
  26         "common_flags",
  27         "common_preempt_count",
  28         "common_pid",
  29         "common_tgid",
  30         FIELD_STRING_IP,
  31         FIELD_STRING_RETIP,
  32         FIELD_STRING_FUNC,
  33 };
  34
  35 /* Printing  in basic type function template */
  36 #define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt)                  \
  37 int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
  38 {                                                                       \
  39         trace_seq_printf(s, fmt, *(type *)data);                        \
  40         return !trace_seq_has_overflowed(s);                            \
  41 }                                                                       \
  42 const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
  43
  44 DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
  45 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
  46 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
  47 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
  48 DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
  49 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
  50 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
  51 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
  52 DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
  53 DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
  54 DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
  55 DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
  56 DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
  57
  58 int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
  59 {
  60         trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
  61         return !trace_seq_has_overflowed(s);
  62 }
  63 const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
  64
  65 /* Print type function for string type */
  66 int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
  67 {
  68         int len = *(u32 *)data >> 16;
  69
  70         if (!len)
  71                 trace_seq_puts(s, FAULT_STRING);
  72         else
  73                 trace_seq_printf(s, "\"%s\"",
  74                                  (const char *)get_loc_data(data, ent));
  75         return !trace_seq_has_overflowed(s);
  76 }
  77
  78 const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
  79
  80 /* Fetch type information table */
  81 static const struct fetch_type probe_fetch_types[] = {
  82         /* Special types */
  83         __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
  84                             "__data_loc char[]"),
  85         __ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
  86                             "__data_loc char[]"),
  87         __ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
  88                             "__data_loc char[]"),
  89         /* Basic types */
  90         ASSIGN_FETCH_TYPE(u8,  u8,  0),
  91         ASSIGN_FETCH_TYPE(u16, u16, 0),
  92         ASSIGN_FETCH_TYPE(u32, u32, 0),
  93         ASSIGN_FETCH_TYPE(u64, u64, 0),
  94         ASSIGN_FETCH_TYPE(s8,  u8,  1),
  95         ASSIGN_FETCH_TYPE(s16, u16, 1),
  96         ASSIGN_FETCH_TYPE(s32, u32, 1),
  97         ASSIGN_FETCH_TYPE(s64, u64, 1),
  98         ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
  99         ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
 100         ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
 101         ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
 102         ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8,  0),
 103         ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
 104
 105         ASSIGN_FETCH_TYPE_END
 106 };
 107
 108 static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
 109 {
 110         int i;
 111
 112         /* Reject the symbol/symstr for uprobes */
 113         if (type && (flags & TPARG_FL_USER) &&
 114             (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
 115                 return NULL;
 116
 117         if (!type)
 118                 type = DEFAULT_FETCH_TYPE_STR;
 119
 120         /* Special case: bitfield */
 121         if (*type == 'b') {
 122                 unsigned long bs;
 123
 124                 type = strchr(type, '/');
 125                 if (!type)
 126                         goto fail;
 127
 128                 type++;
 129                 if (kstrtoul(type, 0, &bs))
 130                         goto fail;
 131
 132                 switch (bs) {
 133                 case 8:
 134                         return find_fetch_type("u8", flags);
 135                 case 16:
 136                         return find_fetch_type("u16", flags);
 137                 case 32:
 138                         return find_fetch_type("u32", flags);
 139                 case 64:
 140                         return find_fetch_type("u64", flags);
 141                 default:
 142                         goto fail;
 143                 }
 144         }
 145
 146         for (i = 0; probe_fetch_types[i].name; i++) {
 147                 if (strcmp(type, probe_fetch_types[i].name) == 0)
 148                         return &probe_fetch_types[i];
 149         }
 150
 151 fail:
 152         return NULL;
 153 }
 154
 155 static struct trace_probe_log trace_probe_log;
 156
 157 void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
 158 {
 159         trace_probe_log.subsystem = subsystem;
 160         trace_probe_log.argc = argc;
 161         trace_probe_log.argv = argv;
 162         trace_probe_log.index = 0;
 163 }
 164
 165 void trace_probe_log_clear(void)
 166 {
 167         memset(&trace_probe_log, 0, sizeof(trace_probe_log));
 168 }
 169
 170 void trace_probe_log_set_index(int index)
 171 {
 172         trace_probe_log.index = index;
 173 }
 174
 175 void __trace_probe_log_err(int offset, int err_type)
 176 {
 177         char *command, *p;
 178         int i, len = 0, pos = 0;
 179
 180         if (!trace_probe_log.argv)
 181                 return;
 182
 183         /* Recalculate the length and allocate buffer */
 184         for (i = 0; i < trace_probe_log.argc; i++) {
 185                 if (i == trace_probe_log.index)
 186                         pos = len;
 187                 len += strlen(trace_probe_log.argv[i]) + 1;
 188         }
 189         command = kzalloc(len, GFP_KERNEL);
 190         if (!command)
 191                 return;
 192
 193         if (trace_probe_log.index >= trace_probe_log.argc) {
 194                 /**
 195                  * Set the error position is next to the last arg + space.
 196                  * Note that len includes the terminal null and the cursor
 197                  * appears at pos + 1.
 198                  */
 199                 pos = len;
 200                 offset = 0;
 201         }
 202
 203         /* And make a command string from argv array */
 204         p = command;
 205         for (i = 0; i < trace_probe_log.argc; i++) {
 206                 len = strlen(trace_probe_log.argv[i]);
 207                 strcpy(p, trace_probe_log.argv[i]);
 208                 p[len] = ' ';
 209                 p += len + 1;
 210         }
 211         *(p - 1) = '\0';
 212
 213         tracing_log_err(NULL, trace_probe_log.subsystem, command,
 214                         trace_probe_err_text, err_type, pos + offset);
 215
 216         kfree(command);
 217 }
 218
 219 /* Split symbol and offset. */
 220 int traceprobe_split_symbol_offset(char *symbol, long *offset)
 221 {
 222         char *tmp;
 223         int ret;
 224
 225         if (!offset)
 226                 return -EINVAL;
 227
 228         tmp = strpbrk(symbol, "+-");
 229         if (tmp) {
 230                 ret = kstrtol(tmp, 0, offset);
 231                 if (ret)
 232                         return ret;
 233                 *tmp = '\0';
 234         } else
 235                 *offset = 0;
 236
 237         return 0;
 238 }
 239
 240 /* @buf must has MAX_EVENT_NAME_LEN size */
 241 int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
 242                                 char *buf, int offset)
 243 {
 244         const char *slash, *event = *pevent;
 245         int len;
 246
 247         slash = strchr(event, '/');
 248         if (!slash)
 249                 slash = strchr(event, '.');
 250
 251         if (slash) {
 252                 if (slash == event) {
 253                         trace_probe_log_err(offset, NO_GROUP_NAME);
 254                         return -EINVAL;
 255                 }
 256                 if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
 257                         trace_probe_log_err(offset, GROUP_TOO_LONG);
 258                         return -EINVAL;
 259                 }
 260                 strscpy(buf, event, slash - event + 1);
 261                 if (!is_good_system_name(buf)) {
 262                         trace_probe_log_err(offset, BAD_GROUP_NAME);
 263                         return -EINVAL;
 264                 }
 265                 *pgroup = buf;
 266                 *pevent = slash + 1;
 267                 offset += slash - event + 1;
 268                 event = *pevent;
 269         }
 270         len = strlen(event);
 271         if (len == 0) {
 272                 if (slash) {
 273                         *pevent = NULL;
 274                         return 0;
 275                 }
 276                 trace_probe_log_err(offset, NO_EVENT_NAME);
 277                 return -EINVAL;
 278         } else if (len > MAX_EVENT_NAME_LEN) {
 279                 trace_probe_log_err(offset, EVENT_TOO_LONG);
 280                 return -EINVAL;
 281         }
 282         if (!is_good_name(event)) {
 283                 trace_probe_log_err(offset, BAD_EVENT_NAME);
 284                 return -EINVAL;
 285         }
 286         return 0;
 287 }
 288
 289 static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
 290                                  struct traceprobe_parse_context *ctx)
 291 {
 292         struct ftrace_event_field *field;
 293         struct list_head *head;
 294
 295         head = trace_get_fields(ctx->event);
 296         list_for_each_entry(field, head, link) {
 297                 if (!strcmp(arg, field->name)) {
 298                         code->op = FETCH_OP_TP_ARG;
 299                         code->data = field;
 300                         return 0;
 301                 }
 302         }
 303         return -ENOENT;
 304 }
 305
 306 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
 307
 308 static u32 btf_type_int(const struct btf_type *t)
 309 {
 310         return *(u32 *)(t + 1);
 311 }
 312
 313 static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
 314 {
 315         const struct btf_type *real_type;
 316         u32 intdata;
 317         s32 tid;
 318
 319         real_type = btf_type_skip_modifiers(btf, type->type, &tid);
 320         if (!real_type)
 321                 return false;
 322
 323         if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
 324                 return false;
 325
 326         intdata = btf_type_int(real_type);
 327         return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
 328                 && BTF_INT_BITS(intdata) == 8;
 329 }
 330
 331 static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
 332 {
 333         const struct btf_type *real_type;
 334         const struct btf_array *array;
 335         u32 intdata;
 336         s32 tid;
 337
 338         if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
 339                 return false;
 340
 341         array = (const struct btf_array *)(type + 1);
 342
 343         real_type = btf_type_skip_modifiers(btf, array->type, &tid);
 344
 345         intdata = btf_type_int(real_type);
 346         return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
 347                 && BTF_INT_BITS(intdata) == 8;
 348 }
 349
 350 static int check_prepare_btf_string_fetch(char *typename,
 351                                 struct fetch_insn **pcode,
 352                                 struct traceprobe_parse_context *ctx)
 353 {
 354         struct btf *btf = ctx->btf;
 355
 356         if (!btf || !ctx->last_type)
 357                 return 0;
 358
 359         /* char [] does not need any change. */
 360         if (btf_type_is_char_array(btf, ctx->last_type))
 361                 return 0;
 362
 363         /* char * requires dereference the pointer. */
 364         if (btf_type_is_char_ptr(btf, ctx->last_type)) {
 365                 struct fetch_insn *code = *pcode + 1;
 366
 367                 if (code->op == FETCH_OP_END) {
 368                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 369                         return -E2BIG;
 370                 }
 371                 if (typename[0] == 'u')
 372                         code->op = FETCH_OP_UDEREF;
 373                 else
 374                         code->op = FETCH_OP_DEREF;
 375                 code->offset = 0;
 376                 *pcode = code;
 377                 return 0;
 378         }
 379         /* Other types are not available for string */
 380         trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
 381         return -EINVAL;
 382 }
 383
 384 static const char *fetch_type_from_btf_type(struct btf *btf,
 385                                         const struct btf_type *type,
 386                                         struct traceprobe_parse_context *ctx)
 387 {
 388         u32 intdata;
 389
 390         /* TODO: const char * could be converted as a string */
 391         switch (BTF_INFO_KIND(type->info)) {
 392         case BTF_KIND_ENUM:
 393                 /* enum is "int", so convert to "s32" */
 394                 return "s32";
 395         case BTF_KIND_ENUM64:
 396                 return "s64";
 397         case BTF_KIND_PTR:
 398                 /* pointer will be converted to "x??" */
 399                 if (IS_ENABLED(CONFIG_64BIT))
 400                         return "x64";
 401                 else
 402                         return "x32";
 403         case BTF_KIND_INT:
 404                 intdata = btf_type_int(type);
 405                 if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
 406                         switch (BTF_INT_BITS(intdata)) {
 407                         case 8:
 408                                 return "s8";
 409                         case 16:
 410                                 return "s16";
 411                         case 32:
 412                                 return "s32";
 413                         case 64:
 414                                 return "s64";
 415                         }
 416                 } else {        /* unsigned */
 417                         switch (BTF_INT_BITS(intdata)) {
 418                         case 8:
 419                                 return "u8";
 420                         case 16:
 421                                 return "u16";
 422                         case 32:
 423                                 return "u32";
 424                         case 64:
 425                                 return "u64";
 426                         }
 427                         /* bitfield, size is encoded in the type */
 428                         ctx->last_bitsize = BTF_INT_BITS(intdata);
 429                         ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
 430                         return "u64";
 431                 }
 432         }
 433         /* TODO: support other types */
 434
 435         return NULL;
 436 }
 437
 438 static int query_btf_context(struct traceprobe_parse_context *ctx)
 439 {
 440         const struct btf_param *param;
 441         const struct btf_type *type;
 442         struct btf *btf;
 443         s32 nr;
 444
 445         if (ctx->btf)
 446                 return 0;
 447
 448         if (!ctx->funcname)
 449                 return -EINVAL;
 450
 451         type = btf_find_func_proto(ctx->funcname, &btf);
 452         if (!type)
 453                 return -ENOENT;
 454
 455         ctx->btf = btf;
 456         ctx->proto = type;
 457
 458         /* ctx->params is optional, since func(void) will not have params. */
 459         nr = 0;
 460         param = btf_get_func_param(type, &nr);
 461         if (!IS_ERR_OR_NULL(param)) {
 462                 /* Hide the first 'data' argument of tracepoint */
 463                 if (ctx->flags & TPARG_FL_TPOINT) {
 464                         nr--;
 465                         param++;
 466                 }
 467         }
 468
 469         if (nr > 0) {
 470                 ctx->nr_params = nr;
 471                 ctx->params = param;
 472         } else {
 473                 ctx->nr_params = 0;
 474                 ctx->params = NULL;
 475         }
 476
 477         return 0;
 478 }
 479
 480 static void clear_btf_context(struct traceprobe_parse_context *ctx)
 481 {
 482         if (ctx->btf) {
 483                 btf_put(ctx->btf);
 484                 ctx->btf = NULL;
 485                 ctx->proto = NULL;
 486                 ctx->params = NULL;
 487                 ctx->nr_params = 0;
 488         }
 489 }
 490
 491 /* Return 1 if the field separater is arrow operator ('->') */
 492 static int split_next_field(char *varname, char **next_field,
 493                             struct traceprobe_parse_context *ctx)
 494 {
 495         char *field;
 496         int ret = 0;
 497
 498         field = strpbrk(varname, ".-");
 499         if (field) {
 500                 if (field[0] == '-' && field[1] == '>') {
 501                         field[0] = '\0';
 502                         field += 2;
 503                         ret = 1;
 504                 } else if (field[0] == '.') {
 505                         field[0] = '\0';
 506                         field += 1;
 507                 } else {
 508                         trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
 509                         return -EINVAL;
 510                 }
 511                 *next_field = field;
 512         }
 513
 514         return ret;
 515 }
 516
 517 /*
 518  * Parse the field of data structure. The @type must be a pointer type
 519  * pointing the target data structure type.
 520  */
 521 static int parse_btf_field(char *fieldname, const struct btf_type *type,
 522                            struct fetch_insn **pcode, struct fetch_insn *end,
 523                            struct traceprobe_parse_context *ctx)
 524 {
 525         struct fetch_insn *code = *pcode;
 526         const struct btf_member *field;
 527         u32 bitoffs, anon_offs;
 528         char *next;
 529         int is_ptr;
 530         s32 tid;
 531
 532         do {
 533                 /* Outer loop for solving arrow operator ('->') */
 534                 if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
 535                         trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
 536                         return -EINVAL;
 537                 }
 538                 /* Convert a struct pointer type to a struct type */
 539                 type = btf_type_skip_modifiers(ctx->btf, type->type, &tid);
 540                 if (!type) {
 541                         trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 542                         return -EINVAL;
 543                 }
 544
 545                 bitoffs = 0;
 546                 do {
 547                         /* Inner loop for solving dot operator ('.') */
 548                         next = NULL;
 549                         is_ptr = split_next_field(fieldname, &next, ctx);
 550                         if (is_ptr < 0)
 551                                 return is_ptr;
 552
 553                         anon_offs = 0;
 554                         field = btf_find_struct_member(ctx->btf, type, fieldname,
 555                                                        &anon_offs);
 556                         if (!field) {
 557                                 trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
 558                                 return -ENOENT;
 559                         }
 560                         /* Add anonymous structure/union offset */
 561                         bitoffs += anon_offs;
 562
 563                         /* Accumulate the bit-offsets of the dot-connected fields */
 564                         if (btf_type_kflag(type)) {
 565                                 bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
 566                                 ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
 567                         } else {
 568                                 bitoffs += field->offset;
 569                                 ctx->last_bitsize = 0;
 570                         }
 571
 572                         type = btf_type_skip_modifiers(ctx->btf, field->type, &tid);
 573                         if (!type) {
 574                                 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 575                                 return -EINVAL;
 576                         }
 577
 578                         ctx->offset += next - fieldname;
 579                         fieldname = next;
 580                 } while (!is_ptr && fieldname);
 581
 582                 if (++code == end) {
 583                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 584                         return -EINVAL;
 585                 }
 586                 code->op = FETCH_OP_DEREF;      /* TODO: user deref support */
 587                 code->offset = bitoffs / 8;
 588                 *pcode = code;
 589
 590                 ctx->last_bitoffs = bitoffs % 8;
 591                 ctx->last_type = type;
 592         } while (fieldname);
 593
 594         return 0;
 595 }
 596
 597 static int parse_btf_arg(char *varname,
 598                          struct fetch_insn **pcode, struct fetch_insn *end,
 599                          struct traceprobe_parse_context *ctx)
 600 {
 601         struct fetch_insn *code = *pcode;
 602         const struct btf_param *params;
 603         const struct btf_type *type;
 604         char *field = NULL;
 605         int i, is_ptr, ret;
 606         u32 tid;
 607
 608         if (WARN_ON_ONCE(!ctx->funcname))
 609                 return -EINVAL;
 610
 611         is_ptr = split_next_field(varname, &field, ctx);
 612         if (is_ptr < 0)
 613                 return is_ptr;
 614         if (!is_ptr && field) {
 615                 /* dot-connected field on an argument is not supported. */
 616                 trace_probe_log_err(ctx->offset + field - varname,
 617                                     NOSUP_DAT_ARG);
 618                 return -EOPNOTSUPP;
 619         }
 620
 621         if (ctx->flags & TPARG_FL_RETURN) {
 622                 if (strcmp(varname, "$retval") != 0) {
 623                         trace_probe_log_err(ctx->offset, NO_BTFARG);
 624                         return -ENOENT;
 625                 }
 626                 code->op = FETCH_OP_RETVAL;
 627                 /* Check whether the function return type is not void */
 628                 if (query_btf_context(ctx) == 0) {
 629                         if (ctx->proto->type == 0) {
 630                                 trace_probe_log_err(ctx->offset, NO_RETVAL);
 631                                 return -ENOENT;
 632                         }
 633                         tid = ctx->proto->type;
 634                         goto found;
 635                 }
 636                 if (field) {
 637                         trace_probe_log_err(ctx->offset + field - varname,
 638                                             NO_BTF_ENTRY);
 639                         return -ENOENT;
 640                 }
 641                 return 0;
 642         }
 643
 644         if (!ctx->btf) {
 645                 ret = query_btf_context(ctx);
 646                 if (ret < 0 || ctx->nr_params == 0) {
 647                         trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
 648                         return PTR_ERR(params);
 649                 }
 650         }
 651         params = ctx->params;
 652
 653         for (i = 0; i < ctx->nr_params; i++) {
 654                 const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
 655
 656                 if (name && !strcmp(name, varname)) {
 657                         code->op = FETCH_OP_ARG;
 658                         if (ctx->flags & TPARG_FL_TPOINT)
 659                                 code->param = i + 1;
 660                         else
 661                                 code->param = i;
 662                         tid = params[i].type;
 663                         goto found;
 664                 }
 665         }
 666         trace_probe_log_err(ctx->offset, NO_BTFARG);
 667         return -ENOENT;
 668
 669 found:
 670         type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
 671         if (!type) {
 672                 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 673                 return -EINVAL;
 674         }
 675         /* Initialize the last type information */
 676         ctx->last_type = type;
 677         ctx->last_bitoffs = 0;
 678         ctx->last_bitsize = 0;
 679         if (field) {
 680                 ctx->offset += field - varname;
 681                 return parse_btf_field(field, type, pcode, end, ctx);
 682         }
 683         return 0;
 684 }
 685
 686 static const struct fetch_type *find_fetch_type_from_btf_type(
 687                                         struct traceprobe_parse_context *ctx)
 688 {
 689         struct btf *btf = ctx->btf;
 690         const char *typestr = NULL;
 691
 692         if (btf && ctx->last_type)
 693                 typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
 694
 695         return find_fetch_type(typestr, ctx->flags);
 696 }
 697
 698 static int parse_btf_bitfield(struct fetch_insn **pcode,
 699                               struct traceprobe_parse_context *ctx)
 700 {
 701         struct fetch_insn *code = *pcode;
 702
 703         if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
 704                 return 0;
 705
 706         code++;
 707         if (code->op != FETCH_OP_NOP) {
 708                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 709                 return -EINVAL;
 710         }
 711         *pcode = code;
 712
 713         code->op = FETCH_OP_MOD_BF;
 714         code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
 715         code->rshift = 64 - ctx->last_bitsize;
 716         code->basesize = 64 / 8;
 717         return 0;
 718 }
 719
 720 #else
 721 static void clear_btf_context(struct traceprobe_parse_context *ctx)
 722 {
 723         ctx->btf = NULL;
 724 }
 725
 726 static int query_btf_context(struct traceprobe_parse_context *ctx)
 727 {
 728         return -EOPNOTSUPP;
 729 }
 730
 731 static int parse_btf_arg(char *varname,
 732                          struct fetch_insn **pcode, struct fetch_insn *end,
 733                          struct traceprobe_parse_context *ctx)
 734 {
 735         trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 736         return -EOPNOTSUPP;
 737 }
 738
 739 static int parse_btf_bitfield(struct fetch_insn **pcode,
 740                               struct traceprobe_parse_context *ctx)
 741 {
 742         trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 743         return -EOPNOTSUPP;
 744 }
 745
 746 #define find_fetch_type_from_btf_type(ctx)              \
 747         find_fetch_type(NULL, ctx->flags)
 748
 749 static int check_prepare_btf_string_fetch(char *typename,
 750                                 struct fetch_insn **pcode,
 751                                 struct traceprobe_parse_context *ctx)
 752 {
 753         return 0;
 754 }
 755
 756 #endif
 757
 758 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
 759
 760 /* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
 761 static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
 762                             struct fetch_insn **pcode,
 763                             struct fetch_insn *end,
 764                             struct traceprobe_parse_context *ctx)
 765 {
 766         struct fetch_insn *code = *pcode;
 767         int err = TP_ERR_BAD_VAR;
 768         char *arg = orig_arg + 1;
 769         unsigned long param;
 770         int ret = 0;
 771         int len;
 772
 773         if (ctx->flags & TPARG_FL_TEVENT) {
 774                 if (code->data)
 775                         return -EFAULT;
 776                 ret = parse_trace_event_arg(arg, code, ctx);
 777                 if (!ret)
 778                         return 0;
 779                 if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 780                         code->op = FETCH_OP_COMM;
 781                         return 0;
 782                 }
 783                 /* backward compatibility */
 784                 ctx->offset = 0;
 785                 goto inval;
 786         }
 787
 788         if (str_has_prefix(arg, "retval")) {
 789                 if (!(ctx->flags & TPARG_FL_RETURN)) {
 790                         err = TP_ERR_RETVAL_ON_PROBE;
 791                         goto inval;
 792                 }
 793                 if (!(ctx->flags & TPARG_FL_KERNEL) ||
 794                     !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
 795                         code->op = FETCH_OP_RETVAL;
 796                         return 0;
 797                 }
 798                 return parse_btf_arg(orig_arg, pcode, end, ctx);
 799         }
 800
 801         len = str_has_prefix(arg, "stack");
 802         if (len) {
 803
 804                 if (arg[len] == '\0') {
 805                         code->op = FETCH_OP_STACKP;
 806                         return 0;
 807                 }
 808
 809                 if (isdigit(arg[len])) {
 810                         ret = kstrtoul(arg + len, 10, &param);
 811                         if (ret)
 812                                 goto inval;
 813
 814                         if ((ctx->flags & TPARG_FL_KERNEL) &&
 815                             param > PARAM_MAX_STACK) {
 816                                 err = TP_ERR_BAD_STACK_NUM;
 817                                 goto inval;
 818                         }
 819                         code->op = FETCH_OP_STACK;
 820                         code->param = (unsigned int)param;
 821                         return 0;
 822                 }
 823                 goto inval;
 824         }
 825
 826         if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 827                 code->op = FETCH_OP_COMM;
 828                 return 0;
 829         }
 830
 831 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 832         len = str_has_prefix(arg, "arg");
 833         if (len && tparg_is_function_entry(ctx->flags)) {
 834                 ret = kstrtoul(arg + len, 10, &param);
 835                 if (ret)
 836                         goto inval;
 837
 838                 if (!param || param > PARAM_MAX_STACK) {
 839                         err = TP_ERR_BAD_ARG_NUM;
 840                         goto inval;
 841                 }
 842
 843                 code->op = FETCH_OP_ARG;
 844                 code->param = (unsigned int)param - 1;
 845                 /*
 846                  * The tracepoint probe will probe a stub function, and the
 847                  * first parameter of the stub is a dummy and should be ignored.
 848                  */
 849                 if (ctx->flags & TPARG_FL_TPOINT)
 850                         code->param++;
 851                 return 0;
 852         }
 853 #endif
 854
 855 inval:
 856         __trace_probe_log_err(ctx->offset, err);
 857         return -EINVAL;
 858 }
 859
 860 static int str_to_immediate(char *str, unsigned long *imm)
 861 {
 862         if (isdigit(str[0]))
 863                 return kstrtoul(str, 0, imm);
 864         else if (str[0] == '-')
 865                 return kstrtol(str, 0, (long *)imm);
 866         else if (str[0] == '+')
 867                 return kstrtol(str + 1, 0, (long *)imm);
 868         return -EINVAL;
 869 }
 870
 871 static int __parse_imm_string(char *str, char **pbuf, int offs)
 872 {
 873         size_t len = strlen(str);
 874
 875         if (str[len - 1] != '"') {
 876                 trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
 877                 return -EINVAL;
 878         }
 879         *pbuf = kstrndup(str, len - 1, GFP_KERNEL);
 880         if (!*pbuf)
 881                 return -ENOMEM;
 882         return 0;
 883 }
 884
 885 /* Recursive argument parser */
 886 static int
 887 parse_probe_arg(char *arg, const struct fetch_type *type,
 888                 struct fetch_insn **pcode, struct fetch_insn *end,
 889                 struct traceprobe_parse_context *ctx)
 890 {
 891         struct fetch_insn *code = *pcode;
 892         unsigned long param;
 893         int deref = FETCH_OP_DEREF;
 894         long offset = 0;
 895         char *tmp;
 896         int ret = 0;
 897
 898         switch (arg[0]) {
 899         case '$':
 900                 ret = parse_probe_vars(arg, type, pcode, end, ctx);
 901                 break;
 902
 903         case '%':       /* named register */
 904                 if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
 905                         /* eprobe and fprobe do not handle registers */
 906                         trace_probe_log_err(ctx->offset, BAD_VAR);
 907                         break;
 908                 }
 909                 ret = regs_query_register_offset(arg + 1);
 910                 if (ret >= 0) {
 911                         code->op = FETCH_OP_REG;
 912                         code->param = (unsigned int)ret;
 913                         ret = 0;
 914                 } else
 915                         trace_probe_log_err(ctx->offset, BAD_REG_NAME);
 916                 break;
 917
 918         case '@':       /* memory, file-offset or symbol */
 919                 if (isdigit(arg[1])) {
 920                         ret = kstrtoul(arg + 1, 0, &param);
 921                         if (ret) {
 922                                 trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
 923                                 break;
 924                         }
 925                         /* load address */
 926                         code->op = FETCH_OP_IMM;
 927                         code->immediate = param;
 928                 } else if (arg[1] == '+') {
 929                         /* kprobes don't support file offsets */
 930                         if (ctx->flags & TPARG_FL_KERNEL) {
 931                                 trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
 932                                 return -EINVAL;
 933                         }
 934                         ret = kstrtol(arg + 2, 0, &offset);
 935                         if (ret) {
 936                                 trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
 937                                 break;
 938                         }
 939
 940                         code->op = FETCH_OP_FOFFS;
 941                         code->immediate = (unsigned long)offset;  // imm64?
 942                 } else {
 943                         /* uprobes don't support symbols */
 944                         if (!(ctx->flags & TPARG_FL_KERNEL)) {
 945                                 trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
 946                                 return -EINVAL;
 947                         }
 948                         /* Preserve symbol for updating */
 949                         code->op = FETCH_NOP_SYMBOL;
 950                         code->data = kstrdup(arg + 1, GFP_KERNEL);
 951                         if (!code->data)
 952                                 return -ENOMEM;
 953                         if (++code == end) {
 954                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 955                                 return -EINVAL;
 956                         }
 957                         code->op = FETCH_OP_IMM;
 958                         code->immediate = 0;
 959                 }
 960                 /* These are fetching from memory */
 961                 if (++code == end) {
 962                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 963                         return -EINVAL;
 964                 }
 965                 *pcode = code;
 966                 code->op = FETCH_OP_DEREF;
 967                 code->offset = offset;
 968                 break;
 969
 970         case '+':       /* deref memory */
 971         case '-':
 972                 if (arg[1] == 'u') {
 973                         deref = FETCH_OP_UDEREF;
 974                         arg[1] = arg[0];
 975                         arg++;
 976                 }
 977                 if (arg[0] == '+')
 978                         arg++;  /* Skip '+', because kstrtol() rejects it. */
 979                 tmp = strchr(arg, '(');
 980                 if (!tmp) {
 981                         trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
 982                         return -EINVAL;
 983                 }
 984                 *tmp = '\0';
 985                 ret = kstrtol(arg, 0, &offset);
 986                 if (ret) {
 987                         trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
 988                         break;
 989                 }
 990                 ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
 991                 arg = tmp + 1;
 992                 tmp = strrchr(arg, ')');
 993                 if (!tmp) {
 994                         trace_probe_log_err(ctx->offset + strlen(arg),
 995                                             DEREF_OPEN_BRACE);
 996                         return -EINVAL;
 997                 } else {
 998                         const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
 999                         int cur_offs = ctx->offset;
1000
1001                         *tmp = '\0';
1002                         ret = parse_probe_arg(arg, t2, &code, end, ctx);
1003                         if (ret)
1004                                 break;
1005                         ctx->offset = cur_offs;
1006                         if (code->op == FETCH_OP_COMM ||
1007                             code->op == FETCH_OP_DATA) {
1008                                 trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1009                                 return -EINVAL;
1010                         }
1011                         if (++code == end) {
1012                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1013                                 return -EINVAL;
1014                         }
1015                         *pcode = code;
1016
1017                         code->op = deref;
1018                         code->offset = offset;
1019                         /* Reset the last type if used */
1020                         ctx->last_type = NULL;
1021                 }
1022                 break;
1023         case '\\':      /* Immediate value */
1024                 if (arg[1] == '"') {    /* Immediate string */
1025                         ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
1026                         if (ret)
1027                                 break;
1028                         code->op = FETCH_OP_DATA;
1029                         code->data = tmp;
1030                 } else {
1031                         ret = str_to_immediate(arg + 1, &code->immediate);
1032                         if (ret)
1033                                 trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1034                         else
1035                                 code->op = FETCH_OP_IMM;
1036                 }
1037                 break;
1038         default:
1039                 if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */
1040                         if (!tparg_is_function_entry(ctx->flags)) {
1041                                 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1042                                 return -EINVAL;
1043                         }
1044                         ret = parse_btf_arg(arg, pcode, end, ctx);
1045                         break;
1046                 }
1047         }
1048         if (!ret && code->op == FETCH_OP_NOP) {
1049                 /* Parsed, but do not find fetch method */
1050                 trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1051                 ret = -EINVAL;
1052         }
1053         return ret;
1054 }
1055
1056 #define BYTES_TO_BITS(nb)       ((BITS_PER_LONG * (nb)) / sizeof(long))
1057
1058 /* Bitfield type needs to be parsed into a fetch function */
1059 static int __parse_bitfield_probe_arg(const char *bf,
1060                                       const struct fetch_type *t,
1061                                       struct fetch_insn **pcode)
1062 {
1063         struct fetch_insn *code = *pcode;
1064         unsigned long bw, bo;
1065         char *tail;
1066
1067         if (*bf != 'b')
1068                 return 0;
1069
1070         bw = simple_strtoul(bf + 1, &tail, 0);  /* Use simple one */
1071
1072         if (bw == 0 || *tail != '@')
1073                 return -EINVAL;
1074
1075         bf = tail + 1;
1076         bo = simple_strtoul(bf, &tail, 0);
1077
1078         if (tail == bf || *tail != '/')
1079                 return -EINVAL;
1080         code++;
1081         if (code->op != FETCH_OP_NOP)
1082                 return -EINVAL;
1083         *pcode = code;
1084
1085         code->op = FETCH_OP_MOD_BF;
1086         code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1087         code->rshift = BYTES_TO_BITS(t->size) - bw;
1088         code->basesize = t->size;
1089
1090         return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1091 }
1092
1093 /* String length checking wrapper */
1094 static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
1095                                            struct probe_arg *parg,
1096                                            struct traceprobe_parse_context *ctx)
1097 {
1098         struct fetch_insn *code, *scode, *tmp = NULL;
1099         char *t, *t2, *t3;
1100         int ret, len;
1101         char *arg;
1102
1103         arg = kstrdup(argv, GFP_KERNEL);
1104         if (!arg)
1105                 return -ENOMEM;
1106
1107         ret = -EINVAL;
1108         len = strlen(arg);
1109         if (len > MAX_ARGSTR_LEN) {
1110                 trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
1111                 goto out;
1112         } else if (len == 0) {
1113                 trace_probe_log_err(ctx->offset, NO_ARG_BODY);
1114                 goto out;
1115         }
1116
1117         ret = -ENOMEM;
1118         parg->comm = kstrdup(arg, GFP_KERNEL);
1119         if (!parg->comm)
1120                 goto out;
1121
1122         ret = -EINVAL;
1123         t = strchr(arg, ':');
1124         if (t) {
1125                 *t = '\0';
1126                 t2 = strchr(++t, '[');
1127                 if (t2) {
1128                         *t2++ = '\0';
1129                         t3 = strchr(t2, ']');
1130                         if (!t3) {
1131                                 int offs = t2 + strlen(t2) - arg;
1132
1133                                 trace_probe_log_err(ctx->offset + offs,
1134                                                     ARRAY_NO_CLOSE);
1135                                 goto out;
1136                         } else if (t3[1] != '\0') {
1137                                 trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1138                                                     BAD_ARRAY_SUFFIX);
1139                                 goto out;
1140                         }
1141                         *t3 = '\0';
1142                         if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
1143                                 trace_probe_log_err(ctx->offset + t2 - arg,
1144                                                     BAD_ARRAY_NUM);
1145                                 goto out;
1146                         }
1147                         if (parg->count > MAX_ARRAY_LEN) {
1148                                 trace_probe_log_err(ctx->offset + t2 - arg,
1149                                                     ARRAY_TOO_BIG);
1150                                 goto out;
1151                         }
1152                 }
1153         }
1154
1155         /*
1156          * Since $comm and immediate string can not be dereferenced,
1157          * we can find those by strcmp. But ignore for eprobes.
1158          */
1159         if (!(ctx->flags & TPARG_FL_TEVENT) &&
1160             (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1161              strncmp(arg, "\\\"", 2) == 0)) {
1162                 /* The type of $comm must be "string", and not an array. */
1163                 if (parg->count || (t && strcmp(t, "string")))
1164                         goto out;
1165                 parg->type = find_fetch_type("string", ctx->flags);
1166         } else
1167                 parg->type = find_fetch_type(t, ctx->flags);
1168         if (!parg->type) {
1169                 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0), BAD_TYPE);
1170                 goto out;
1171         }
1172         parg->offset = *size;
1173         *size += parg->type->size * (parg->count ?: 1);
1174
1175         ret = -ENOMEM;
1176         if (parg->count) {
1177                 len = strlen(parg->type->fmttype) + 6;
1178                 parg->fmt = kmalloc(len, GFP_KERNEL);
1179                 if (!parg->fmt)
1180                         goto out;
1181                 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
1182                          parg->count);
1183         }
1184
1185         code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL);
1186         if (!code)
1187                 goto out;
1188         code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
1189
1190         ctx->last_type = NULL;
1191         ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
1192                               ctx);
1193         if (ret)
1194                 goto fail;
1195
1196         /* Update storing type if BTF is available */
1197         if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1198             ctx->last_type) {
1199                 if (!t) {
1200                         parg->type = find_fetch_type_from_btf_type(ctx);
1201                 } else if (strstr(t, "string")) {
1202                         ret = check_prepare_btf_string_fetch(t, &code, ctx);
1203                         if (ret)
1204                                 goto fail;
1205                 }
1206         }
1207
1208         ret = -EINVAL;
1209         /* Store operation */
1210         if (parg->type->is_string) {
1211                 if (!strcmp(parg->type->name, "symstr")) {
1212                         if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1213                             code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1214                             code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1215                                 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1216                                                     BAD_SYMSTRING);
1217                                 goto fail;
1218                         }
1219                 } else {
1220                         if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1221                             code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1222                             code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
1223                                 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1224                                                     BAD_STRING);
1225                                 goto fail;
1226                         }
1227                 }
1228                 if (!strcmp(parg->type->name, "symstr") ||
1229                     (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1230                      code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
1231                      parg->count) {
1232                         /*
1233                          * IMM, DATA and COMM is pointing actual address, those
1234                          * must be kept, and if parg->count != 0, this is an
1235                          * array of string pointers instead of string address
1236                          * itself.
1237                          * For the symstr, it doesn't need to dereference, thus
1238                          * it just get the value.
1239                          */
1240                         code++;
1241                         if (code->op != FETCH_OP_NOP) {
1242                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1243                                 goto fail;
1244                         }
1245                 }
1246                 /* If op == DEREF, replace it with STRING */
1247                 if (!strcmp(parg->type->name, "ustring") ||
1248                     code->op == FETCH_OP_UDEREF)
1249                         code->op = FETCH_OP_ST_USTRING;
1250                 else if (!strcmp(parg->type->name, "symstr"))
1251                         code->op = FETCH_OP_ST_SYMSTR;
1252                 else
1253                         code->op = FETCH_OP_ST_STRING;
1254                 code->size = parg->type->size;
1255                 parg->dynamic = true;
1256         } else if (code->op == FETCH_OP_DEREF) {
1257                 code->op = FETCH_OP_ST_MEM;
1258                 code->size = parg->type->size;
1259         } else if (code->op == FETCH_OP_UDEREF) {
1260                 code->op = FETCH_OP_ST_UMEM;
1261                 code->size = parg->type->size;
1262         } else {
1263                 code++;
1264                 if (code->op != FETCH_OP_NOP) {
1265                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1266                         goto fail;
1267                 }
1268                 code->op = FETCH_OP_ST_RAW;
1269                 code->size = parg->type->size;
1270         }
1271         scode = code;
1272         /* Modify operation */
1273         if (t != NULL) {
1274                 ret = __parse_bitfield_probe_arg(t, parg->type, &code);
1275                 if (ret) {
1276                         trace_probe_log_err(ctx->offset + t - arg, BAD_BITFIELD);
1277                         goto fail;
1278                 }
1279         } else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1280                    ctx->last_type) {
1281                 ret = parse_btf_bitfield(&code, ctx);
1282                 if (ret)
1283                         goto fail;
1284         }
1285         ret = -EINVAL;
1286         /* Loop(Array) operation */
1287         if (parg->count) {
1288                 if (scode->op != FETCH_OP_ST_MEM &&
1289                     scode->op != FETCH_OP_ST_STRING &&
1290                     scode->op != FETCH_OP_ST_USTRING) {
1291                         trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1292                                             BAD_STRING);
1293                         goto fail;
1294                 }
1295                 code++;
1296                 if (code->op != FETCH_OP_NOP) {
1297                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1298                         goto fail;
1299                 }
1300                 code->op = FETCH_OP_LP_ARRAY;
1301                 code->param = parg->count;
1302         }
1303         code++;
1304         code->op = FETCH_OP_END;
1305
1306         ret = 0;
1307         /* Shrink down the code buffer */
1308         parg->code = kcalloc(code - tmp + 1, sizeof(*code), GFP_KERNEL);
1309         if (!parg->code)
1310                 ret = -ENOMEM;
1311         else
1312                 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
1313
1314 fail:
1315         if (ret) {
1316                 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
1317                         if (code->op == FETCH_NOP_SYMBOL ||
1318                             code->op == FETCH_OP_DATA)
1319                                 kfree(code->data);
1320         }
1321         kfree(tmp);
1322 out:
1323         kfree(arg);
1324
1325         return ret;
1326 }
1327
1328 /* Return 1 if name is reserved or already used by another argument */
1329 static int traceprobe_conflict_field_name(const char *name,
1330                                           struct probe_arg *args, int narg)
1331 {
1332         int i;
1333
1334         for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
1335                 if (strcmp(reserved_field_names[i], name) == 0)
1336                         return 1;
1337
1338         for (i = 0; i < narg; i++)
1339                 if (strcmp(args[i].name, name) == 0)
1340                         return 1;
1341
1342         return 0;
1343 }
1344
1345 static char *generate_probe_arg_name(const char *arg, int idx)
1346 {
1347         char *name = NULL;
1348         const char *end;
1349
1350         /*
1351          * If argument name is omitted, try arg as a name (BTF variable)
1352          * or "argN".
1353          */
1354         if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1355                 end = strchr(arg, ':');
1356                 if (!end)
1357                         end = arg + strlen(arg);
1358
1359                 name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
1360                 if (!name || !is_good_name(name)) {
1361                         kfree(name);
1362                         name = NULL;
1363                 }
1364         }
1365
1366         if (!name)
1367                 name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
1368
1369         return name;
1370 }
1371
1372 int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
1373                                struct traceprobe_parse_context *ctx)
1374 {
1375         struct probe_arg *parg = &tp->args[i];
1376         const char *body;
1377
1378         /* Increment count for freeing args in error case */
1379         tp->nr_args++;
1380
1381         body = strchr(arg, '=');
1382         if (body) {
1383                 if (body - arg > MAX_ARG_NAME_LEN) {
1384                         trace_probe_log_err(0, ARG_NAME_TOO_LONG);
1385                         return -EINVAL;
1386                 } else if (body == arg) {
1387                         trace_probe_log_err(0, NO_ARG_NAME);
1388                         return -EINVAL;
1389                 }
1390                 parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
1391                 body++;
1392         } else {
1393                 parg->name = generate_probe_arg_name(arg, i);
1394                 body = arg;
1395         }
1396         if (!parg->name)
1397                 return -ENOMEM;
1398
1399         if (!is_good_name(parg->name)) {
1400                 trace_probe_log_err(0, BAD_ARG_NAME);
1401                 return -EINVAL;
1402         }
1403         if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
1404                 trace_probe_log_err(0, USED_ARG_NAME);
1405                 return -EINVAL;
1406         }
1407         ctx->offset = body - arg;
1408         /* Parse fetch argument */
1409         return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
1410 }
1411
1412 void traceprobe_free_probe_arg(struct probe_arg *arg)
1413 {
1414         struct fetch_insn *code = arg->code;
1415
1416         while (code && code->op != FETCH_OP_END) {
1417                 if (code->op == FETCH_NOP_SYMBOL ||
1418                     code->op == FETCH_OP_DATA)
1419                         kfree(code->data);
1420                 code++;
1421         }
1422         kfree(arg->code);
1423         kfree(arg->name);
1424         kfree(arg->comm);
1425         kfree(arg->fmt);
1426 }
1427
1428 static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
1429                             struct traceprobe_parse_context *ctx)
1430 {
1431         int i, found = 0;
1432
1433         for (i = 0; i < argc; i++)
1434                 if (str_has_prefix(argv[i], "$arg")) {
1435                         trace_probe_log_set_index(i + 2);
1436
1437                         if (!tparg_is_function_entry(ctx->flags)) {
1438                                 trace_probe_log_err(0, NOFENTRY_ARGS);
1439                                 return -EINVAL;
1440                         }
1441
1442                         if (isdigit(argv[i][4])) {
1443                                 found = 1;
1444                                 continue;
1445                         }
1446
1447                         if (argv[i][4] != '*') {
1448                                 trace_probe_log_err(0, BAD_VAR);
1449                                 return -EINVAL;
1450                         }
1451
1452                         if (*args_idx >= 0 && *args_idx < argc) {
1453                                 trace_probe_log_err(0, DOUBLE_ARGS);
1454                                 return -EINVAL;
1455                         }
1456                         found = 1;
1457                         *args_idx = i;
1458                 }
1459
1460         return found;
1461 }
1462
1463 static int sprint_nth_btf_arg(int idx, const char *type,
1464                               char *buf, int bufsize,
1465                               struct traceprobe_parse_context *ctx)
1466 {
1467         const char *name;
1468         int ret;
1469
1470         if (idx >= ctx->nr_params) {
1471                 trace_probe_log_err(0, NO_BTFARG);
1472                 return -ENOENT;
1473         }
1474         name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off);
1475         if (!name) {
1476                 trace_probe_log_err(0, NO_BTF_ENTRY);
1477                 return -ENOENT;
1478         }
1479         ret = snprintf(buf, bufsize, "%s%s", name, type);
1480         if (ret >= bufsize) {
1481                 trace_probe_log_err(0, ARGS_2LONG);
1482                 return -E2BIG;
1483         }
1484         return ret;
1485 }
1486
1487 /* Return new_argv which must be freed after use */
1488 const char **traceprobe_expand_meta_args(int argc, const char *argv[],
1489                                          int *new_argc, char *buf, int bufsize,
1490                                          struct traceprobe_parse_context *ctx)
1491 {
1492         const struct btf_param *params = NULL;
1493         int i, j, n, used, ret, args_idx = -1;
1494         const char **new_argv = NULL;
1495
1496         ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
1497         if (ret < 0)
1498                 return ERR_PTR(ret);
1499
1500         if (!ret) {
1501                 *new_argc = argc;
1502                 return NULL;
1503         }
1504
1505         ret = query_btf_context(ctx);
1506         if (ret < 0 || ctx->nr_params == 0) {
1507                 if (args_idx != -1) {
1508                         /* $arg* requires BTF info */
1509                         trace_probe_log_err(0, NOSUP_BTFARG);
1510                         return (const char **)params;
1511                 }
1512                 *new_argc = argc;
1513                 return NULL;
1514         }
1515
1516         if (args_idx >= 0)
1517                 *new_argc = argc + ctx->nr_params - 1;
1518         else
1519                 *new_argc = argc;
1520
1521         new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL);
1522         if (!new_argv)
1523                 return ERR_PTR(-ENOMEM);
1524
1525         used = 0;
1526         for (i = 0, j = 0; i < argc; i++) {
1527                 trace_probe_log_set_index(i + 2);
1528                 if (i == args_idx) {
1529                         for (n = 0; n < ctx->nr_params; n++) {
1530                                 ret = sprint_nth_btf_arg(n, "", buf + used,
1531                                                          bufsize - used, ctx);
1532                                 if (ret < 0)
1533                                         goto error;
1534
1535                                 new_argv[j++] = buf + used;
1536                                 used += ret + 1;
1537                         }
1538                         continue;
1539                 }
1540
1541                 if (str_has_prefix(argv[i], "$arg")) {
1542                         char *type = NULL;
1543
1544                         n = simple_strtoul(argv[i] + 4, &type, 10);
1545                         if (type && !(*type == ':' || *type == '\0')) {
1546                                 trace_probe_log_err(0, BAD_VAR);
1547                                 ret = -ENOENT;
1548                                 goto error;
1549                         }
1550                         /* Note: $argN starts from $arg1 */
1551                         ret = sprint_nth_btf_arg(n - 1, type, buf + used,
1552                                                  bufsize - used, ctx);
1553                         if (ret < 0)
1554                                 goto error;
1555                         new_argv[j++] = buf + used;
1556                         used += ret + 1;
1557                 } else
1558                         new_argv[j++] = argv[i];
1559         }
1560
1561         return new_argv;
1562
1563 error:
1564         kfree(new_argv);
1565         return ERR_PTR(ret);
1566 }
1567
1568 void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
1569 {
1570         clear_btf_context(ctx);
1571 }
1572
1573 int traceprobe_update_arg(struct probe_arg *arg)
1574 {
1575         struct fetch_insn *code = arg->code;
1576         long offset;
1577         char *tmp;
1578         char c;
1579         int ret = 0;
1580
1581         while (code && code->op != FETCH_OP_END) {
1582                 if (code->op == FETCH_NOP_SYMBOL) {
1583                         if (code[1].op != FETCH_OP_IMM)
1584                                 return -EINVAL;
1585
1586                         tmp = strpbrk(code->data, "+-");
1587                         if (tmp)
1588                                 c = *tmp;
1589                         ret = traceprobe_split_symbol_offset(code->data,
1590                                                              &offset);
1591                         if (ret)
1592                                 return ret;
1593
1594                         code[1].immediate =
1595                                 (unsigned long)kallsyms_lookup_name(code->data);
1596                         if (tmp)
1597                                 *tmp = c;
1598                         if (!code[1].immediate)
1599                                 return -ENOENT;
1600                         code[1].immediate += offset;
1601                 }
1602                 code++;
1603         }
1604         return 0;
1605 }
1606
1607 /* When len=0, we just calculate the needed length */
1608 #define LEN_OR_ZERO (len ? len - pos : 0)
1609 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
1610                            enum probe_print_type ptype)
1611 {
1612         struct probe_arg *parg;
1613         int i, j;
1614         int pos = 0;
1615         const char *fmt, *arg;
1616
1617         switch (ptype) {
1618         case PROBE_PRINT_NORMAL:
1619                 fmt = "(%lx)";
1620                 arg = ", REC->" FIELD_STRING_IP;
1621                 break;
1622         case PROBE_PRINT_RETURN:
1623                 fmt = "(%lx <- %lx)";
1624                 arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1625                 break;
1626         case PROBE_PRINT_EVENT:
1627                 fmt = "";
1628                 arg = "";
1629                 break;
1630         default:
1631                 WARN_ON_ONCE(1);
1632                 return 0;
1633         }
1634
1635         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1636
1637         for (i = 0; i < tp->nr_args; i++) {
1638                 parg = tp->args + i;
1639                 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
1640                 if (parg->count) {
1641                         pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
1642                                         parg->type->fmt);
1643                         for (j = 1; j < parg->count; j++)
1644                                 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
1645                                                 parg->type->fmt);
1646                         pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
1647                 } else
1648                         pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
1649                                         parg->type->fmt);
1650         }
1651
1652         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
1653
1654         for (i = 0; i < tp->nr_args; i++) {
1655                 parg = tp->args + i;
1656                 if (parg->count) {
1657                         if (parg->type->is_string)
1658                                 fmt = ", __get_str(%s[%d])";
1659                         else
1660                                 fmt = ", REC->%s[%d]";
1661                         for (j = 0; j < parg->count; j++)
1662                                 pos += snprintf(buf + pos, LEN_OR_ZERO,
1663                                                 fmt, parg->name, j);
1664                 } else {
1665                         if (parg->type->is_string)
1666                                 fmt = ", __get_str(%s)";
1667                         else
1668                                 fmt = ", REC->%s";
1669                         pos += snprintf(buf + pos, LEN_OR_ZERO,
1670                                         fmt, parg->name);
1671                 }
1672         }
1673
1674         /* return the length of print_fmt */
1675         return pos;
1676 }
1677 #undef LEN_OR_ZERO
1678
1679 int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
1680 {
1681         struct trace_event_call *call = trace_probe_event_call(tp);
1682         int len;
1683         char *print_fmt;
1684
1685         /* First: called with 0 length to calculate the needed length */
1686         len = __set_print_fmt(tp, NULL, 0, ptype);
1687         print_fmt = kmalloc(len + 1, GFP_KERNEL);
1688         if (!print_fmt)
1689                 return -ENOMEM;
1690
1691         /* Second: actually write the @print_fmt */
1692         __set_print_fmt(tp, print_fmt, len + 1, ptype);
1693         call->print_fmt = print_fmt;
1694
1695         return 0;
1696 }
1697
1698 int traceprobe_define_arg_fields(struct trace_event_call *event_call,
1699                                  size_t offset, struct trace_probe *tp)
1700 {
1701         int ret, i;
1702
1703         /* Set argument names as fields */
1704         for (i = 0; i < tp->nr_args; i++) {
1705                 struct probe_arg *parg = &tp->args[i];
1706                 const char *fmt = parg->type->fmttype;
1707                 int size = parg->type->size;
1708
1709                 if (parg->fmt)
1710                         fmt = parg->fmt;
1711                 if (parg->count)
1712                         size *= parg->count;
1713                 ret = trace_define_field(event_call, fmt, parg->name,
1714                                          offset + parg->offset, size,
1715                                          parg->type->is_signed,
1716                                          FILTER_OTHER);
1717                 if (ret)
1718                         return ret;
1719         }
1720         return 0;
1721 }
1722
1723 static void trace_probe_event_free(struct trace_probe_event *tpe)
1724 {
1725         kfree(tpe->class.system);
1726         kfree(tpe->call.name);
1727         kfree(tpe->call.print_fmt);
1728         kfree(tpe);
1729 }
1730
1731 int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1732 {
1733         if (trace_probe_has_sibling(tp))
1734                 return -EBUSY;
1735
1736         list_del_init(&tp->list);
1737         trace_probe_event_free(tp->event);
1738
1739         tp->event = to->event;
1740         list_add_tail(&tp->list, trace_probe_probe_list(to));
1741
1742         return 0;
1743 }
1744
1745 void trace_probe_unlink(struct trace_probe *tp)
1746 {
1747         list_del_init(&tp->list);
1748         if (list_empty(trace_probe_probe_list(tp)))
1749                 trace_probe_event_free(tp->event);
1750         tp->event = NULL;
1751 }
1752
1753 void trace_probe_cleanup(struct trace_probe *tp)
1754 {
1755         int i;
1756
1757         for (i = 0; i < tp->nr_args; i++)
1758                 traceprobe_free_probe_arg(&tp->args[i]);
1759
1760         if (tp->event)
1761                 trace_probe_unlink(tp);
1762 }
1763
1764 int trace_probe_init(struct trace_probe *tp, const char *event,
1765                      const char *group, bool alloc_filter)
1766 {
1767         struct trace_event_call *call;
1768         size_t size = sizeof(struct trace_probe_event);
1769         int ret = 0;
1770
1771         if (!event || !group)
1772                 return -EINVAL;
1773
1774         if (alloc_filter)
1775                 size += sizeof(struct trace_uprobe_filter);
1776
1777         tp->event = kzalloc(size, GFP_KERNEL);
1778         if (!tp->event)
1779                 return -ENOMEM;
1780
1781         INIT_LIST_HEAD(&tp->event->files);
1782         INIT_LIST_HEAD(&tp->event->class.fields);
1783         INIT_LIST_HEAD(&tp->event->probes);
1784         INIT_LIST_HEAD(&tp->list);
1785         list_add(&tp->list, &tp->event->probes);
1786
1787         call = trace_probe_event_call(tp);
1788         call->class = &tp->event->class;
1789         call->name = kstrdup(event, GFP_KERNEL);
1790         if (!call->name) {
1791                 ret = -ENOMEM;
1792                 goto error;
1793         }
1794
1795         tp->event->class.system = kstrdup(group, GFP_KERNEL);
1796         if (!tp->event->class.system) {
1797                 ret = -ENOMEM;
1798                 goto error;
1799         }
1800
1801         return 0;
1802
1803 error:
1804         trace_probe_cleanup(tp);
1805         return ret;
1806 }
1807
1808 static struct trace_event_call *
1809 find_trace_event_call(const char *system, const char *event_name)
1810 {
1811         struct trace_event_call *tp_event;
1812         const char *name;
1813
1814         list_for_each_entry(tp_event, &ftrace_events, list) {
1815                 if (!tp_event->class->system ||
1816                     strcmp(system, tp_event->class->system))
1817                         continue;
1818                 name = trace_event_name(tp_event);
1819                 if (!name || strcmp(event_name, name))
1820                         continue;
1821                 return tp_event;
1822         }
1823
1824         return NULL;
1825 }
1826
1827 int trace_probe_register_event_call(struct trace_probe *tp)
1828 {
1829         struct trace_event_call *call = trace_probe_event_call(tp);
1830         int ret;
1831
1832         lockdep_assert_held(&event_mutex);
1833
1834         if (find_trace_event_call(trace_probe_group_name(tp),
1835                                   trace_probe_name(tp)))
1836                 return -EEXIST;
1837
1838         ret = register_trace_event(&call->event);
1839         if (!ret)
1840                 return -ENODEV;
1841
1842         ret = trace_add_event_call(call);
1843         if (ret)
1844                 unregister_trace_event(&call->event);
1845
1846         return ret;
1847 }
1848
1849 int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
1850 {
1851         struct event_file_link *link;
1852
1853         link = kmalloc(sizeof(*link), GFP_KERNEL);
1854         if (!link)
1855                 return -ENOMEM;
1856
1857         link->file = file;
1858         INIT_LIST_HEAD(&link->list);
1859         list_add_tail_rcu(&link->list, &tp->event->files);
1860         trace_probe_set_flag(tp, TP_FLAG_TRACE);
1861         return 0;
1862 }
1863
1864 struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
1865                                                   struct trace_event_file *file)
1866 {
1867         struct event_file_link *link;
1868
1869         trace_probe_for_each_link(link, tp) {
1870                 if (link->file == file)
1871                         return link;
1872         }
1873
1874         return NULL;
1875 }
1876
1877 int trace_probe_remove_file(struct trace_probe *tp,
1878                             struct trace_event_file *file)
1879 {
1880         struct event_file_link *link;
1881
1882         link = trace_probe_get_file_link(tp, file);
1883         if (!link)
1884                 return -ENOENT;
1885
1886         list_del_rcu(&link->list);
1887         kvfree_rcu_mightsleep(link);
1888
1889         if (list_empty(&tp->event->files))
1890                 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
1891
1892         return 0;
1893 }
1894
1895 /*
1896  * Return the smallest index of different type argument (start from 1).
1897  * If all argument types and name are same, return 0.
1898  */
1899 int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
1900 {
1901         int i;
1902
1903         /* In case of more arguments */
1904         if (a->nr_args < b->nr_args)
1905                 return a->nr_args + 1;
1906         if (a->nr_args > b->nr_args)
1907                 return b->nr_args + 1;
1908
1909         for (i = 0; i < a->nr_args; i++) {
1910                 if ((b->nr_args <= i) ||
1911                     ((a->args[i].type != b->args[i].type) ||
1912                      (a->args[i].count != b->args[i].count) ||
1913                      strcmp(a->args[i].name, b->args[i].name)))
1914                         return i + 1;
1915         }
1916
1917         return 0;
1918 }
1919
1920 bool trace_probe_match_command_args(struct trace_probe *tp,
1921                                     int argc, const char **argv)
1922 {
1923         char buf[MAX_ARGSTR_LEN + 1];
1924         int i;
1925
1926         if (tp->nr_args < argc)
1927                 return false;
1928
1929         for (i = 0; i < argc; i++) {
1930                 snprintf(buf, sizeof(buf), "%s=%s",
1931                          tp->args[i].name, tp->args[i].comm);
1932                 if (strcmp(buf, argv[i]))
1933                         return false;
1934         }
1935         return true;
1936 }
1937
1938 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
1939 {
1940         int argc = 0, ret = 0;
1941         char **argv;
1942
1943         argv = argv_split(GFP_KERNEL, raw_command, &argc);
1944         if (!argv)
1945                 return -ENOMEM;
1946
1947         if (argc)
1948                 ret = createfn(argc, (const char **)argv);
1949
1950         argv_free(argv);
1951
1952         return ret;
1953 }
1954
1955 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
1956                  u8 *data, void *field)
1957 {
1958         void *p;
1959         int i, j;
1960
1961         for (i = 0; i < nr_args; i++) {
1962                 struct probe_arg *a = args + i;
1963
1964                 trace_seq_printf(s, " %s=", a->name);
1965                 if (likely(!a->count)) {
1966                         if (!a->type->print(s, data + a->offset, field))
1967                                 return -ENOMEM;
1968                         continue;
1969                 }
1970                 trace_seq_putc(s, '{');
1971                 p = data + a->offset;
1972                 for (j = 0; j < a->count; j++) {
1973                         if (!a->type->print(s, p, field))
1974                                 return -ENOMEM;
1975                         trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
1976                         p += a->type->size;
1977                 }
1978         }
1979         return 0;
1980 }