Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

at v4.7-rc3 1735 lines 40 kB view raw
1/* 2 * bpf-loader.c 3 * 4 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 5 * Copyright (C) 2015 Huawei Inc. 6 */ 7 8#include <linux/bpf.h> 9#include <bpf/libbpf.h> 10#include <bpf/bpf.h> 11#include <linux/err.h> 12#include <linux/string.h> 13#include "perf.h" 14#include "debug.h" 15#include "bpf-loader.h" 16#include "bpf-prologue.h" 17#include "llvm-utils.h" 18#include "probe-event.h" 19#include "probe-finder.h" // for MAX_PROBES 20#include "parse-events.h" 21#include "llvm-utils.h" 22 23#define DEFINE_PRINT_FN(name, level) \ 24static int libbpf_##name(const char *fmt, ...) \ 25{ \ 26 va_list args; \ 27 int ret; \ 28 \ 29 va_start(args, fmt); \ 30 ret = veprintf(level, verbose, pr_fmt(fmt), args);\ 31 va_end(args); \ 32 return ret; \ 33} 34 35DEFINE_PRINT_FN(warning, 1) 36DEFINE_PRINT_FN(info, 1) 37DEFINE_PRINT_FN(debug, 1) 38 39struct bpf_prog_priv { 40 struct perf_probe_event pev; 41 bool need_prologue; 42 struct bpf_insn *insns_buf; 43 int nr_types; 44 int *type_mapping; 45}; 46 47static bool libbpf_initialized; 48 49struct bpf_object * 50bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name) 51{ 52 struct bpf_object *obj; 53 54 if (!libbpf_initialized) { 55 libbpf_set_print(libbpf_warning, 56 libbpf_info, 57 libbpf_debug); 58 libbpf_initialized = true; 59 } 60 61 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name); 62 if (IS_ERR(obj)) { 63 pr_debug("bpf: failed to load buffer\n"); 64 return ERR_PTR(-EINVAL); 65 } 66 67 return obj; 68} 69 70struct bpf_object *bpf__prepare_load(const char *filename, bool source) 71{ 72 struct bpf_object *obj; 73 74 if (!libbpf_initialized) { 75 libbpf_set_print(libbpf_warning, 76 libbpf_info, 77 libbpf_debug); 78 libbpf_initialized = true; 79 } 80 81 if (source) { 82 int err; 83 void *obj_buf; 84 size_t obj_buf_sz; 85 86 err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz); 87 if (err) 88 return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE); 89 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename); 90 free(obj_buf); 91 } else 92 obj = bpf_object__open(filename); 93 94 if (IS_ERR(obj)) { 95 pr_debug("bpf: failed to load %s\n", filename); 96 return obj; 97 } 98 99 return obj; 100} 101 102void bpf__clear(void) 103{ 104 struct bpf_object *obj, *tmp; 105 106 bpf_object__for_each_safe(obj, tmp) { 107 bpf__unprobe(obj); 108 bpf_object__close(obj); 109 } 110} 111 112static void 113clear_prog_priv(struct bpf_program *prog __maybe_unused, 114 void *_priv) 115{ 116 struct bpf_prog_priv *priv = _priv; 117 118 cleanup_perf_probe_events(&priv->pev, 1); 119 zfree(&priv->insns_buf); 120 zfree(&priv->type_mapping); 121 free(priv); 122} 123 124static int 125prog_config__exec(const char *value, struct perf_probe_event *pev) 126{ 127 pev->uprobes = true; 128 pev->target = strdup(value); 129 if (!pev->target) 130 return -ENOMEM; 131 return 0; 132} 133 134static int 135prog_config__module(const char *value, struct perf_probe_event *pev) 136{ 137 pev->uprobes = false; 138 pev->target = strdup(value); 139 if (!pev->target) 140 return -ENOMEM; 141 return 0; 142} 143 144static int 145prog_config__bool(const char *value, bool *pbool, bool invert) 146{ 147 int err; 148 bool bool_value; 149 150 if (!pbool) 151 return -EINVAL; 152 153 err = strtobool(value, &bool_value); 154 if (err) 155 return err; 156 157 *pbool = invert ? !bool_value : bool_value; 158 return 0; 159} 160 161static int 162prog_config__inlines(const char *value, 163 struct perf_probe_event *pev __maybe_unused) 164{ 165 return prog_config__bool(value, &probe_conf.no_inlines, true); 166} 167 168static int 169prog_config__force(const char *value, 170 struct perf_probe_event *pev __maybe_unused) 171{ 172 return prog_config__bool(value, &probe_conf.force_add, false); 173} 174 175static struct { 176 const char *key; 177 const char *usage; 178 const char *desc; 179 int (*func)(const char *, struct perf_probe_event *); 180} bpf_prog_config_terms[] = { 181 { 182 .key = "exec", 183 .usage = "exec=<full path of file>", 184 .desc = "Set uprobe target", 185 .func = prog_config__exec, 186 }, 187 { 188 .key = "module", 189 .usage = "module=<module name> ", 190 .desc = "Set kprobe module", 191 .func = prog_config__module, 192 }, 193 { 194 .key = "inlines", 195 .usage = "inlines=[yes|no] ", 196 .desc = "Probe at inline symbol", 197 .func = prog_config__inlines, 198 }, 199 { 200 .key = "force", 201 .usage = "force=[yes|no] ", 202 .desc = "Forcibly add events with existing name", 203 .func = prog_config__force, 204 }, 205}; 206 207static int 208do_prog_config(const char *key, const char *value, 209 struct perf_probe_event *pev) 210{ 211 unsigned int i; 212 213 pr_debug("config bpf program: %s=%s\n", key, value); 214 for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++) 215 if (strcmp(key, bpf_prog_config_terms[i].key) == 0) 216 return bpf_prog_config_terms[i].func(value, pev); 217 218 pr_debug("BPF: ERROR: invalid program config option: %s=%s\n", 219 key, value); 220 221 pr_debug("\nHint: Valid options are:\n"); 222 for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++) 223 pr_debug("\t%s:\t%s\n", bpf_prog_config_terms[i].usage, 224 bpf_prog_config_terms[i].desc); 225 pr_debug("\n"); 226 227 return -BPF_LOADER_ERRNO__PROGCONF_TERM; 228} 229 230static const char * 231parse_prog_config_kvpair(const char *config_str, struct perf_probe_event *pev) 232{ 233 char *text = strdup(config_str); 234 char *sep, *line; 235 const char *main_str = NULL; 236 int err = 0; 237 238 if (!text) { 239 pr_debug("No enough memory: dup config_str failed\n"); 240 return ERR_PTR(-ENOMEM); 241 } 242 243 line = text; 244 while ((sep = strchr(line, ';'))) { 245 char *equ; 246 247 *sep = '\0'; 248 equ = strchr(line, '='); 249 if (!equ) { 250 pr_warning("WARNING: invalid config in BPF object: %s\n", 251 line); 252 pr_warning("\tShould be 'key=value'.\n"); 253 goto nextline; 254 } 255 *equ = '\0'; 256 257 err = do_prog_config(line, equ + 1, pev); 258 if (err) 259 break; 260nextline: 261 line = sep + 1; 262 } 263 264 if (!err) 265 main_str = config_str + (line - text); 266 free(text); 267 268 return err ? ERR_PTR(err) : main_str; 269} 270 271static int 272parse_prog_config(const char *config_str, struct perf_probe_event *pev) 273{ 274 int err; 275 const char *main_str = parse_prog_config_kvpair(config_str, pev); 276 277 if (IS_ERR(main_str)) 278 return PTR_ERR(main_str); 279 280 err = parse_perf_probe_command(main_str, pev); 281 if (err < 0) { 282 pr_debug("bpf: '%s' is not a valid config string\n", 283 config_str); 284 /* parse failed, don't need clear pev. */ 285 return -BPF_LOADER_ERRNO__CONFIG; 286 } 287 return 0; 288} 289 290static int 291config_bpf_program(struct bpf_program *prog) 292{ 293 struct perf_probe_event *pev = NULL; 294 struct bpf_prog_priv *priv = NULL; 295 const char *config_str; 296 int err; 297 298 /* Initialize per-program probing setting */ 299 probe_conf.no_inlines = false; 300 probe_conf.force_add = false; 301 302 config_str = bpf_program__title(prog, false); 303 if (IS_ERR(config_str)) { 304 pr_debug("bpf: unable to get title for program\n"); 305 return PTR_ERR(config_str); 306 } 307 308 priv = calloc(sizeof(*priv), 1); 309 if (!priv) { 310 pr_debug("bpf: failed to alloc priv\n"); 311 return -ENOMEM; 312 } 313 pev = &priv->pev; 314 315 pr_debug("bpf: config program '%s'\n", config_str); 316 err = parse_prog_config(config_str, pev); 317 if (err) 318 goto errout; 319 320 if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) { 321 pr_debug("bpf: '%s': group for event is set and not '%s'.\n", 322 config_str, PERF_BPF_PROBE_GROUP); 323 err = -BPF_LOADER_ERRNO__GROUP; 324 goto errout; 325 } else if (!pev->group) 326 pev->group = strdup(PERF_BPF_PROBE_GROUP); 327 328 if (!pev->group) { 329 pr_debug("bpf: strdup failed\n"); 330 err = -ENOMEM; 331 goto errout; 332 } 333 334 if (!pev->event) { 335 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n", 336 config_str); 337 err = -BPF_LOADER_ERRNO__EVENTNAME; 338 goto errout; 339 } 340 pr_debug("bpf: config '%s' is ok\n", config_str); 341 342 err = bpf_program__set_private(prog, priv, clear_prog_priv); 343 if (err) { 344 pr_debug("Failed to set priv for program '%s'\n", config_str); 345 goto errout; 346 } 347 348 return 0; 349 350errout: 351 if (pev) 352 clear_perf_probe_event(pev); 353 free(priv); 354 return err; 355} 356 357static int bpf__prepare_probe(void) 358{ 359 static int err = 0; 360 static bool initialized = false; 361 362 /* 363 * Make err static, so if init failed the first, bpf__prepare_probe() 364 * fails each time without calling init_probe_symbol_maps multiple 365 * times. 366 */ 367 if (initialized) 368 return err; 369 370 initialized = true; 371 err = init_probe_symbol_maps(false); 372 if (err < 0) 373 pr_debug("Failed to init_probe_symbol_maps\n"); 374 probe_conf.max_probes = MAX_PROBES; 375 return err; 376} 377 378static int 379preproc_gen_prologue(struct bpf_program *prog, int n, 380 struct bpf_insn *orig_insns, int orig_insns_cnt, 381 struct bpf_prog_prep_result *res) 382{ 383 struct probe_trace_event *tev; 384 struct perf_probe_event *pev; 385 struct bpf_prog_priv *priv; 386 struct bpf_insn *buf; 387 size_t prologue_cnt = 0; 388 int i, err; 389 390 err = bpf_program__get_private(prog, (void **)&priv); 391 if (err || !priv) 392 goto errout; 393 394 pev = &priv->pev; 395 396 if (n < 0 || n >= priv->nr_types) 397 goto errout; 398 399 /* Find a tev belongs to that type */ 400 for (i = 0; i < pev->ntevs; i++) { 401 if (priv->type_mapping[i] == n) 402 break; 403 } 404 405 if (i >= pev->ntevs) { 406 pr_debug("Internal error: prologue type %d not found\n", n); 407 return -BPF_LOADER_ERRNO__PROLOGUE; 408 } 409 410 tev = &pev->tevs[i]; 411 412 buf = priv->insns_buf; 413 err = bpf__gen_prologue(tev->args, tev->nargs, 414 buf, &prologue_cnt, 415 BPF_MAXINSNS - orig_insns_cnt); 416 if (err) { 417 const char *title; 418 419 title = bpf_program__title(prog, false); 420 if (!title) 421 title = "[unknown]"; 422 423 pr_debug("Failed to generate prologue for program %s\n", 424 title); 425 return err; 426 } 427 428 memcpy(&buf[prologue_cnt], orig_insns, 429 sizeof(struct bpf_insn) * orig_insns_cnt); 430 431 res->new_insn_ptr = buf; 432 res->new_insn_cnt = prologue_cnt + orig_insns_cnt; 433 res->pfd = NULL; 434 return 0; 435 436errout: 437 pr_debug("Internal error in preproc_gen_prologue\n"); 438 return -BPF_LOADER_ERRNO__PROLOGUE; 439} 440 441/* 442 * compare_tev_args is reflexive, transitive and antisymmetric. 443 * I can proof it but this margin is too narrow to contain. 444 */ 445static int compare_tev_args(const void *ptev1, const void *ptev2) 446{ 447 int i, ret; 448 const struct probe_trace_event *tev1 = 449 *(const struct probe_trace_event **)ptev1; 450 const struct probe_trace_event *tev2 = 451 *(const struct probe_trace_event **)ptev2; 452 453 ret = tev2->nargs - tev1->nargs; 454 if (ret) 455 return ret; 456 457 for (i = 0; i < tev1->nargs; i++) { 458 struct probe_trace_arg *arg1, *arg2; 459 struct probe_trace_arg_ref *ref1, *ref2; 460 461 arg1 = &tev1->args[i]; 462 arg2 = &tev2->args[i]; 463 464 ret = strcmp(arg1->value, arg2->value); 465 if (ret) 466 return ret; 467 468 ref1 = arg1->ref; 469 ref2 = arg2->ref; 470 471 while (ref1 && ref2) { 472 ret = ref2->offset - ref1->offset; 473 if (ret) 474 return ret; 475 476 ref1 = ref1->next; 477 ref2 = ref2->next; 478 } 479 480 if (ref1 || ref2) 481 return ref2 ? 1 : -1; 482 } 483 484 return 0; 485} 486 487/* 488 * Assign a type number to each tevs in a pev. 489 * mapping is an array with same slots as tevs in that pev. 490 * nr_types will be set to number of types. 491 */ 492static int map_prologue(struct perf_probe_event *pev, int *mapping, 493 int *nr_types) 494{ 495 int i, type = 0; 496 struct probe_trace_event **ptevs; 497 498 size_t array_sz = sizeof(*ptevs) * pev->ntevs; 499 500 ptevs = malloc(array_sz); 501 if (!ptevs) { 502 pr_debug("No ehough memory: alloc ptevs failed\n"); 503 return -ENOMEM; 504 } 505 506 pr_debug("In map_prologue, ntevs=%d\n", pev->ntevs); 507 for (i = 0; i < pev->ntevs; i++) 508 ptevs[i] = &pev->tevs[i]; 509 510 qsort(ptevs, pev->ntevs, sizeof(*ptevs), 511 compare_tev_args); 512 513 for (i = 0; i < pev->ntevs; i++) { 514 int n; 515 516 n = ptevs[i] - pev->tevs; 517 if (i == 0) { 518 mapping[n] = type; 519 pr_debug("mapping[%d]=%d\n", n, type); 520 continue; 521 } 522 523 if (compare_tev_args(ptevs + i, ptevs + i - 1) == 0) 524 mapping[n] = type; 525 else 526 mapping[n] = ++type; 527 528 pr_debug("mapping[%d]=%d\n", n, mapping[n]); 529 } 530 free(ptevs); 531 *nr_types = type + 1; 532 533 return 0; 534} 535 536static int hook_load_preprocessor(struct bpf_program *prog) 537{ 538 struct perf_probe_event *pev; 539 struct bpf_prog_priv *priv; 540 bool need_prologue = false; 541 int err, i; 542 543 err = bpf_program__get_private(prog, (void **)&priv); 544 if (err || !priv) { 545 pr_debug("Internal error when hook preprocessor\n"); 546 return -BPF_LOADER_ERRNO__INTERNAL; 547 } 548 549 pev = &priv->pev; 550 for (i = 0; i < pev->ntevs; i++) { 551 struct probe_trace_event *tev = &pev->tevs[i]; 552 553 if (tev->nargs > 0) { 554 need_prologue = true; 555 break; 556 } 557 } 558 559 /* 560 * Since all tevs don't have argument, we don't need generate 561 * prologue. 562 */ 563 if (!need_prologue) { 564 priv->need_prologue = false; 565 return 0; 566 } 567 568 priv->need_prologue = true; 569 priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS); 570 if (!priv->insns_buf) { 571 pr_debug("No enough memory: alloc insns_buf failed\n"); 572 return -ENOMEM; 573 } 574 575 priv->type_mapping = malloc(sizeof(int) * pev->ntevs); 576 if (!priv->type_mapping) { 577 pr_debug("No enough memory: alloc type_mapping failed\n"); 578 return -ENOMEM; 579 } 580 memset(priv->type_mapping, -1, 581 sizeof(int) * pev->ntevs); 582 583 err = map_prologue(pev, priv->type_mapping, &priv->nr_types); 584 if (err) 585 return err; 586 587 err = bpf_program__set_prep(prog, priv->nr_types, 588 preproc_gen_prologue); 589 return err; 590} 591 592int bpf__probe(struct bpf_object *obj) 593{ 594 int err = 0; 595 struct bpf_program *prog; 596 struct bpf_prog_priv *priv; 597 struct perf_probe_event *pev; 598 599 err = bpf__prepare_probe(); 600 if (err) { 601 pr_debug("bpf__prepare_probe failed\n"); 602 return err; 603 } 604 605 bpf_object__for_each_program(prog, obj) { 606 err = config_bpf_program(prog); 607 if (err) 608 goto out; 609 610 err = bpf_program__get_private(prog, (void **)&priv); 611 if (err || !priv) 612 goto out; 613 pev = &priv->pev; 614 615 err = convert_perf_probe_events(pev, 1); 616 if (err < 0) { 617 pr_debug("bpf_probe: failed to convert perf probe events"); 618 goto out; 619 } 620 621 err = apply_perf_probe_events(pev, 1); 622 if (err < 0) { 623 pr_debug("bpf_probe: failed to apply perf probe events"); 624 goto out; 625 } 626 627 /* 628 * After probing, let's consider prologue, which 629 * adds program fetcher to BPF programs. 630 * 631 * hook_load_preprocessorr() hooks pre-processor 632 * to bpf_program, let it generate prologue 633 * dynamically during loading. 634 */ 635 err = hook_load_preprocessor(prog); 636 if (err) 637 goto out; 638 } 639out: 640 return err < 0 ? err : 0; 641} 642 643#define EVENTS_WRITE_BUFSIZE 4096 644int bpf__unprobe(struct bpf_object *obj) 645{ 646 int err, ret = 0; 647 struct bpf_program *prog; 648 struct bpf_prog_priv *priv; 649 650 bpf_object__for_each_program(prog, obj) { 651 int i; 652 653 err = bpf_program__get_private(prog, (void **)&priv); 654 if (err || !priv) 655 continue; 656 657 for (i = 0; i < priv->pev.ntevs; i++) { 658 struct probe_trace_event *tev = &priv->pev.tevs[i]; 659 char name_buf[EVENTS_WRITE_BUFSIZE]; 660 struct strfilter *delfilter; 661 662 snprintf(name_buf, EVENTS_WRITE_BUFSIZE, 663 "%s:%s", tev->group, tev->event); 664 name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0'; 665 666 delfilter = strfilter__new(name_buf, NULL); 667 if (!delfilter) { 668 pr_debug("Failed to create filter for unprobing\n"); 669 ret = -ENOMEM; 670 continue; 671 } 672 673 err = del_perf_probe_events(delfilter); 674 strfilter__delete(delfilter); 675 if (err) { 676 pr_debug("Failed to delete %s\n", name_buf); 677 ret = err; 678 continue; 679 } 680 } 681 } 682 return ret; 683} 684 685int bpf__load(struct bpf_object *obj) 686{ 687 int err; 688 689 err = bpf_object__load(obj); 690 if (err) { 691 pr_debug("bpf: load objects failed\n"); 692 return err; 693 } 694 return 0; 695} 696 697int bpf__foreach_tev(struct bpf_object *obj, 698 bpf_prog_iter_callback_t func, 699 void *arg) 700{ 701 struct bpf_program *prog; 702 int err; 703 704 bpf_object__for_each_program(prog, obj) { 705 struct probe_trace_event *tev; 706 struct perf_probe_event *pev; 707 struct bpf_prog_priv *priv; 708 int i, fd; 709 710 err = bpf_program__get_private(prog, 711 (void **)&priv); 712 if (err || !priv) { 713 pr_debug("bpf: failed to get private field\n"); 714 return -BPF_LOADER_ERRNO__INTERNAL; 715 } 716 717 pev = &priv->pev; 718 for (i = 0; i < pev->ntevs; i++) { 719 tev = &pev->tevs[i]; 720 721 if (priv->need_prologue) { 722 int type = priv->type_mapping[i]; 723 724 fd = bpf_program__nth_fd(prog, type); 725 } else { 726 fd = bpf_program__fd(prog); 727 } 728 729 if (fd < 0) { 730 pr_debug("bpf: failed to get file descriptor\n"); 731 return fd; 732 } 733 734 err = (*func)(tev, fd, arg); 735 if (err) { 736 pr_debug("bpf: call back failed, stop iterate\n"); 737 return err; 738 } 739 } 740 } 741 return 0; 742} 743 744enum bpf_map_op_type { 745 BPF_MAP_OP_SET_VALUE, 746 BPF_MAP_OP_SET_EVSEL, 747}; 748 749enum bpf_map_key_type { 750 BPF_MAP_KEY_ALL, 751 BPF_MAP_KEY_RANGES, 752}; 753 754struct bpf_map_op { 755 struct list_head list; 756 enum bpf_map_op_type op_type; 757 enum bpf_map_key_type key_type; 758 union { 759 struct parse_events_array array; 760 } k; 761 union { 762 u64 value; 763 struct perf_evsel *evsel; 764 } v; 765}; 766 767struct bpf_map_priv { 768 struct list_head ops_list; 769}; 770 771static void 772bpf_map_op__delete(struct bpf_map_op *op) 773{ 774 if (!list_empty(&op->list)) 775 list_del(&op->list); 776 if (op->key_type == BPF_MAP_KEY_RANGES) 777 parse_events__clear_array(&op->k.array); 778 free(op); 779} 780 781static void 782bpf_map_priv__purge(struct bpf_map_priv *priv) 783{ 784 struct bpf_map_op *pos, *n; 785 786 list_for_each_entry_safe(pos, n, &priv->ops_list, list) { 787 list_del_init(&pos->list); 788 bpf_map_op__delete(pos); 789 } 790} 791 792static void 793bpf_map_priv__clear(struct bpf_map *map __maybe_unused, 794 void *_priv) 795{ 796 struct bpf_map_priv *priv = _priv; 797 798 bpf_map_priv__purge(priv); 799 free(priv); 800} 801 802static int 803bpf_map_op_setkey(struct bpf_map_op *op, struct parse_events_term *term) 804{ 805 op->key_type = BPF_MAP_KEY_ALL; 806 if (!term) 807 return 0; 808 809 if (term->array.nr_ranges) { 810 size_t memsz = term->array.nr_ranges * 811 sizeof(op->k.array.ranges[0]); 812 813 op->k.array.ranges = memdup(term->array.ranges, memsz); 814 if (!op->k.array.ranges) { 815 pr_debug("No enough memory to alloc indices for map\n"); 816 return -ENOMEM; 817 } 818 op->key_type = BPF_MAP_KEY_RANGES; 819 op->k.array.nr_ranges = term->array.nr_ranges; 820 } 821 return 0; 822} 823 824static struct bpf_map_op * 825bpf_map_op__new(struct parse_events_term *term) 826{ 827 struct bpf_map_op *op; 828 int err; 829 830 op = zalloc(sizeof(*op)); 831 if (!op) { 832 pr_debug("Failed to alloc bpf_map_op\n"); 833 return ERR_PTR(-ENOMEM); 834 } 835 INIT_LIST_HEAD(&op->list); 836 837 err = bpf_map_op_setkey(op, term); 838 if (err) { 839 free(op); 840 return ERR_PTR(err); 841 } 842 return op; 843} 844 845static struct bpf_map_op * 846bpf_map_op__clone(struct bpf_map_op *op) 847{ 848 struct bpf_map_op *newop; 849 850 newop = memdup(op, sizeof(*op)); 851 if (!newop) { 852 pr_debug("Failed to alloc bpf_map_op\n"); 853 return NULL; 854 } 855 856 INIT_LIST_HEAD(&newop->list); 857 if (op->key_type == BPF_MAP_KEY_RANGES) { 858 size_t memsz = op->k.array.nr_ranges * 859 sizeof(op->k.array.ranges[0]); 860 861 newop->k.array.ranges = memdup(op->k.array.ranges, memsz); 862 if (!newop->k.array.ranges) { 863 pr_debug("Failed to alloc indices for map\n"); 864 free(newop); 865 return NULL; 866 } 867 } 868 869 return newop; 870} 871 872static struct bpf_map_priv * 873bpf_map_priv__clone(struct bpf_map_priv *priv) 874{ 875 struct bpf_map_priv *newpriv; 876 struct bpf_map_op *pos, *newop; 877 878 newpriv = zalloc(sizeof(*newpriv)); 879 if (!newpriv) { 880 pr_debug("No enough memory to alloc map private\n"); 881 return NULL; 882 } 883 INIT_LIST_HEAD(&newpriv->ops_list); 884 885 list_for_each_entry(pos, &priv->ops_list, list) { 886 newop = bpf_map_op__clone(pos); 887 if (!newop) { 888 bpf_map_priv__purge(newpriv); 889 return NULL; 890 } 891 list_add_tail(&newop->list, &newpriv->ops_list); 892 } 893 894 return newpriv; 895} 896 897static int 898bpf_map__add_op(struct bpf_map *map, struct bpf_map_op *op) 899{ 900 struct bpf_map_priv *priv; 901 const char *map_name; 902 int err; 903 904 map_name = bpf_map__get_name(map); 905 err = bpf_map__get_private(map, (void **)&priv); 906 if (err) { 907 pr_debug("Failed to get private from map %s\n", map_name); 908 return err; 909 } 910 911 if (!priv) { 912 priv = zalloc(sizeof(*priv)); 913 if (!priv) { 914 pr_debug("No enough memory to alloc map private\n"); 915 return -ENOMEM; 916 } 917 INIT_LIST_HEAD(&priv->ops_list); 918 919 if (bpf_map__set_private(map, priv, bpf_map_priv__clear)) { 920 free(priv); 921 return -BPF_LOADER_ERRNO__INTERNAL; 922 } 923 } 924 925 list_add_tail(&op->list, &priv->ops_list); 926 return 0; 927} 928 929static struct bpf_map_op * 930bpf_map__add_newop(struct bpf_map *map, struct parse_events_term *term) 931{ 932 struct bpf_map_op *op; 933 int err; 934 935 op = bpf_map_op__new(term); 936 if (IS_ERR(op)) 937 return op; 938 939 err = bpf_map__add_op(map, op); 940 if (err) { 941 bpf_map_op__delete(op); 942 return ERR_PTR(err); 943 } 944 return op; 945} 946 947static int 948__bpf_map__config_value(struct bpf_map *map, 949 struct parse_events_term *term) 950{ 951 struct bpf_map_def def; 952 struct bpf_map_op *op; 953 const char *map_name; 954 int err; 955 956 map_name = bpf_map__get_name(map); 957 958 err = bpf_map__get_def(map, &def); 959 if (err) { 960 pr_debug("Unable to get map definition from '%s'\n", 961 map_name); 962 return -BPF_LOADER_ERRNO__INTERNAL; 963 } 964 965 if (def.type != BPF_MAP_TYPE_ARRAY) { 966 pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n", 967 map_name); 968 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE; 969 } 970 if (def.key_size < sizeof(unsigned int)) { 971 pr_debug("Map %s has incorrect key size\n", map_name); 972 return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE; 973 } 974 switch (def.value_size) { 975 case 1: 976 case 2: 977 case 4: 978 case 8: 979 break; 980 default: 981 pr_debug("Map %s has incorrect value size\n", map_name); 982 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE; 983 } 984 985 op = bpf_map__add_newop(map, term); 986 if (IS_ERR(op)) 987 return PTR_ERR(op); 988 op->op_type = BPF_MAP_OP_SET_VALUE; 989 op->v.value = term->val.num; 990 return 0; 991} 992 993static int 994bpf_map__config_value(struct bpf_map *map, 995 struct parse_events_term *term, 996 struct perf_evlist *evlist __maybe_unused) 997{ 998 if (!term->err_val) { 999 pr_debug("Config value not set\n"); 1000 return -BPF_LOADER_ERRNO__OBJCONF_CONF; 1001 } 1002 1003 if (term->type_val != PARSE_EVENTS__TERM_TYPE_NUM) { 1004 pr_debug("ERROR: wrong value type for 'value'\n"); 1005 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE; 1006 } 1007 1008 return __bpf_map__config_value(map, term); 1009} 1010 1011static int 1012__bpf_map__config_event(struct bpf_map *map, 1013 struct parse_events_term *term, 1014 struct perf_evlist *evlist) 1015{ 1016 struct perf_evsel *evsel; 1017 struct bpf_map_def def; 1018 struct bpf_map_op *op; 1019 const char *map_name; 1020 int err; 1021 1022 map_name = bpf_map__get_name(map); 1023 evsel = perf_evlist__find_evsel_by_str(evlist, term->val.str); 1024 if (!evsel) { 1025 pr_debug("Event (for '%s') '%s' doesn't exist\n", 1026 map_name, term->val.str); 1027 return -BPF_LOADER_ERRNO__OBJCONF_MAP_NOEVT; 1028 } 1029 1030 err = bpf_map__get_def(map, &def); 1031 if (err) { 1032 pr_debug("Unable to get map definition from '%s'\n", 1033 map_name); 1034 return err; 1035 } 1036 1037 /* 1038 * No need to check key_size and value_size: 1039 * kernel has already checked them. 1040 */ 1041 if (def.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 1042 pr_debug("Map %s type is not BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 1043 map_name); 1044 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE; 1045 } 1046 1047 op = bpf_map__add_newop(map, term); 1048 if (IS_ERR(op)) 1049 return PTR_ERR(op); 1050 op->op_type = BPF_MAP_OP_SET_EVSEL; 1051 op->v.evsel = evsel; 1052 return 0; 1053} 1054 1055static int 1056bpf_map__config_event(struct bpf_map *map, 1057 struct parse_events_term *term, 1058 struct perf_evlist *evlist) 1059{ 1060 if (!term->err_val) { 1061 pr_debug("Config value not set\n"); 1062 return -BPF_LOADER_ERRNO__OBJCONF_CONF; 1063 } 1064 1065 if (term->type_val != PARSE_EVENTS__TERM_TYPE_STR) { 1066 pr_debug("ERROR: wrong value type for 'event'\n"); 1067 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE; 1068 } 1069 1070 return __bpf_map__config_event(map, term, evlist); 1071} 1072 1073struct bpf_obj_config__map_func { 1074 const char *config_opt; 1075 int (*config_func)(struct bpf_map *, struct parse_events_term *, 1076 struct perf_evlist *); 1077}; 1078 1079struct bpf_obj_config__map_func bpf_obj_config__map_funcs[] = { 1080 {"value", bpf_map__config_value}, 1081 {"event", bpf_map__config_event}, 1082}; 1083 1084static int 1085config_map_indices_range_check(struct parse_events_term *term, 1086 struct bpf_map *map, 1087 const char *map_name) 1088{ 1089 struct parse_events_array *array = &term->array; 1090 struct bpf_map_def def; 1091 unsigned int i; 1092 int err; 1093 1094 if (!array->nr_ranges) 1095 return 0; 1096 if (!array->ranges) { 1097 pr_debug("ERROR: map %s: array->nr_ranges is %d but range array is NULL\n", 1098 map_name, (int)array->nr_ranges); 1099 return -BPF_LOADER_ERRNO__INTERNAL; 1100 } 1101 1102 err = bpf_map__get_def(map, &def); 1103 if (err) { 1104 pr_debug("ERROR: Unable to get map definition from '%s'\n", 1105 map_name); 1106 return -BPF_LOADER_ERRNO__INTERNAL; 1107 } 1108 1109 for (i = 0; i < array->nr_ranges; i++) { 1110 unsigned int start = array->ranges[i].start; 1111 size_t length = array->ranges[i].length; 1112 unsigned int idx = start + length - 1; 1113 1114 if (idx >= def.max_entries) { 1115 pr_debug("ERROR: index %d too large\n", idx); 1116 return -BPF_LOADER_ERRNO__OBJCONF_MAP_IDX2BIG; 1117 } 1118 } 1119 return 0; 1120} 1121 1122static int 1123bpf__obj_config_map(struct bpf_object *obj, 1124 struct parse_events_term *term, 1125 struct perf_evlist *evlist, 1126 int *key_scan_pos) 1127{ 1128 /* key is "map:<mapname>.<config opt>" */ 1129 char *map_name = strdup(term->config + sizeof("map:") - 1); 1130 struct bpf_map *map; 1131 int err = -BPF_LOADER_ERRNO__OBJCONF_OPT; 1132 char *map_opt; 1133 size_t i; 1134 1135 if (!map_name) 1136 return -ENOMEM; 1137 1138 map_opt = strchr(map_name, '.'); 1139 if (!map_opt) { 1140 pr_debug("ERROR: Invalid map config: %s\n", map_name); 1141 goto out; 1142 } 1143 1144 *map_opt++ = '\0'; 1145 if (*map_opt == '\0') { 1146 pr_debug("ERROR: Invalid map option: %s\n", term->config); 1147 goto out; 1148 } 1149 1150 map = bpf_object__get_map_by_name(obj, map_name); 1151 if (!map) { 1152 pr_debug("ERROR: Map %s doesn't exist\n", map_name); 1153 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST; 1154 goto out; 1155 } 1156 1157 *key_scan_pos += strlen(map_opt); 1158 err = config_map_indices_range_check(term, map, map_name); 1159 if (err) 1160 goto out; 1161 *key_scan_pos -= strlen(map_opt); 1162 1163 for (i = 0; i < ARRAY_SIZE(bpf_obj_config__map_funcs); i++) { 1164 struct bpf_obj_config__map_func *func = 1165 &bpf_obj_config__map_funcs[i]; 1166 1167 if (strcmp(map_opt, func->config_opt) == 0) { 1168 err = func->config_func(map, term, evlist); 1169 goto out; 1170 } 1171 } 1172 1173 pr_debug("ERROR: Invalid map config option '%s'\n", map_opt); 1174 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT; 1175out: 1176 free(map_name); 1177 if (!err) 1178 key_scan_pos += strlen(map_opt); 1179 return err; 1180} 1181 1182int bpf__config_obj(struct bpf_object *obj, 1183 struct parse_events_term *term, 1184 struct perf_evlist *evlist, 1185 int *error_pos) 1186{ 1187 int key_scan_pos = 0; 1188 int err; 1189 1190 if (!obj || !term || !term->config) 1191 return -EINVAL; 1192 1193 if (!prefixcmp(term->config, "map:")) { 1194 key_scan_pos = sizeof("map:") - 1; 1195 err = bpf__obj_config_map(obj, term, evlist, &key_scan_pos); 1196 goto out; 1197 } 1198 err = -BPF_LOADER_ERRNO__OBJCONF_OPT; 1199out: 1200 if (error_pos) 1201 *error_pos = key_scan_pos; 1202 return err; 1203 1204} 1205 1206typedef int (*map_config_func_t)(const char *name, int map_fd, 1207 struct bpf_map_def *pdef, 1208 struct bpf_map_op *op, 1209 void *pkey, void *arg); 1210 1211static int 1212foreach_key_array_all(map_config_func_t func, 1213 void *arg, const char *name, 1214 int map_fd, struct bpf_map_def *pdef, 1215 struct bpf_map_op *op) 1216{ 1217 unsigned int i; 1218 int err; 1219 1220 for (i = 0; i < pdef->max_entries; i++) { 1221 err = func(name, map_fd, pdef, op, &i, arg); 1222 if (err) { 1223 pr_debug("ERROR: failed to insert value to %s[%u]\n", 1224 name, i); 1225 return err; 1226 } 1227 } 1228 return 0; 1229} 1230 1231static int 1232foreach_key_array_ranges(map_config_func_t func, void *arg, 1233 const char *name, int map_fd, 1234 struct bpf_map_def *pdef, 1235 struct bpf_map_op *op) 1236{ 1237 unsigned int i, j; 1238 int err; 1239 1240 for (i = 0; i < op->k.array.nr_ranges; i++) { 1241 unsigned int start = op->k.array.ranges[i].start; 1242 size_t length = op->k.array.ranges[i].length; 1243 1244 for (j = 0; j < length; j++) { 1245 unsigned int idx = start + j; 1246 1247 err = func(name, map_fd, pdef, op, &idx, arg); 1248 if (err) { 1249 pr_debug("ERROR: failed to insert value to %s[%u]\n", 1250 name, idx); 1251 return err; 1252 } 1253 } 1254 } 1255 return 0; 1256} 1257 1258static int 1259bpf_map_config_foreach_key(struct bpf_map *map, 1260 map_config_func_t func, 1261 void *arg) 1262{ 1263 int err, map_fd; 1264 const char *name; 1265 struct bpf_map_op *op; 1266 struct bpf_map_def def; 1267 struct bpf_map_priv *priv; 1268 1269 name = bpf_map__get_name(map); 1270 1271 err = bpf_map__get_private(map, (void **)&priv); 1272 if (err) { 1273 pr_debug("ERROR: failed to get private from map %s\n", name); 1274 return -BPF_LOADER_ERRNO__INTERNAL; 1275 } 1276 if (!priv || list_empty(&priv->ops_list)) { 1277 pr_debug("INFO: nothing to config for map %s\n", name); 1278 return 0; 1279 } 1280 1281 err = bpf_map__get_def(map, &def); 1282 if (err) { 1283 pr_debug("ERROR: failed to get definition from map %s\n", name); 1284 return -BPF_LOADER_ERRNO__INTERNAL; 1285 } 1286 map_fd = bpf_map__get_fd(map); 1287 if (map_fd < 0) { 1288 pr_debug("ERROR: failed to get fd from map %s\n", name); 1289 return map_fd; 1290 } 1291 1292 list_for_each_entry(op, &priv->ops_list, list) { 1293 switch (def.type) { 1294 case BPF_MAP_TYPE_ARRAY: 1295 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 1296 switch (op->key_type) { 1297 case BPF_MAP_KEY_ALL: 1298 err = foreach_key_array_all(func, arg, name, 1299 map_fd, &def, op); 1300 break; 1301 case BPF_MAP_KEY_RANGES: 1302 err = foreach_key_array_ranges(func, arg, name, 1303 map_fd, &def, 1304 op); 1305 break; 1306 default: 1307 pr_debug("ERROR: keytype for map '%s' invalid\n", 1308 name); 1309 return -BPF_LOADER_ERRNO__INTERNAL; 1310 } 1311 if (err) 1312 return err; 1313 break; 1314 default: 1315 pr_debug("ERROR: type of '%s' incorrect\n", name); 1316 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE; 1317 } 1318 } 1319 1320 return 0; 1321} 1322 1323static int 1324apply_config_value_for_key(int map_fd, void *pkey, 1325 size_t val_size, u64 val) 1326{ 1327 int err = 0; 1328 1329 switch (val_size) { 1330 case 1: { 1331 u8 _val = (u8)(val); 1332 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY); 1333 break; 1334 } 1335 case 2: { 1336 u16 _val = (u16)(val); 1337 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY); 1338 break; 1339 } 1340 case 4: { 1341 u32 _val = (u32)(val); 1342 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY); 1343 break; 1344 } 1345 case 8: { 1346 err = bpf_map_update_elem(map_fd, pkey, &val, BPF_ANY); 1347 break; 1348 } 1349 default: 1350 pr_debug("ERROR: invalid value size\n"); 1351 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE; 1352 } 1353 if (err && errno) 1354 err = -errno; 1355 return err; 1356} 1357 1358static int 1359apply_config_evsel_for_key(const char *name, int map_fd, void *pkey, 1360 struct perf_evsel *evsel) 1361{ 1362 struct xyarray *xy = evsel->fd; 1363 struct perf_event_attr *attr; 1364 unsigned int key, events; 1365 bool check_pass = false; 1366 int *evt_fd; 1367 int err; 1368 1369 if (!xy) { 1370 pr_debug("ERROR: evsel not ready for map %s\n", name); 1371 return -BPF_LOADER_ERRNO__INTERNAL; 1372 } 1373 1374 if (xy->row_size / xy->entry_size != 1) { 1375 pr_debug("ERROR: Dimension of target event is incorrect for map %s\n", 1376 name); 1377 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM; 1378 } 1379 1380 attr = &evsel->attr; 1381 if (attr->inherit) { 1382 pr_debug("ERROR: Can't put inherit event into map %s\n", name); 1383 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH; 1384 } 1385 1386 if (perf_evsel__is_bpf_output(evsel)) 1387 check_pass = true; 1388 if (attr->type == PERF_TYPE_RAW) 1389 check_pass = true; 1390 if (attr->type == PERF_TYPE_HARDWARE) 1391 check_pass = true; 1392 if (!check_pass) { 1393 pr_debug("ERROR: Event type is wrong for map %s\n", name); 1394 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE; 1395 } 1396 1397 events = xy->entries / (xy->row_size / xy->entry_size); 1398 key = *((unsigned int *)pkey); 1399 if (key >= events) { 1400 pr_debug("ERROR: there is no event %d for map %s\n", 1401 key, name); 1402 return -BPF_LOADER_ERRNO__OBJCONF_MAP_MAPSIZE; 1403 } 1404 evt_fd = xyarray__entry(xy, key, 0); 1405 err = bpf_map_update_elem(map_fd, pkey, evt_fd, BPF_ANY); 1406 if (err && errno) 1407 err = -errno; 1408 return err; 1409} 1410 1411static int 1412apply_obj_config_map_for_key(const char *name, int map_fd, 1413 struct bpf_map_def *pdef __maybe_unused, 1414 struct bpf_map_op *op, 1415 void *pkey, void *arg __maybe_unused) 1416{ 1417 int err; 1418 1419 switch (op->op_type) { 1420 case BPF_MAP_OP_SET_VALUE: 1421 err = apply_config_value_for_key(map_fd, pkey, 1422 pdef->value_size, 1423 op->v.value); 1424 break; 1425 case BPF_MAP_OP_SET_EVSEL: 1426 err = apply_config_evsel_for_key(name, map_fd, pkey, 1427 op->v.evsel); 1428 break; 1429 default: 1430 pr_debug("ERROR: unknown value type for '%s'\n", name); 1431 err = -BPF_LOADER_ERRNO__INTERNAL; 1432 } 1433 return err; 1434} 1435 1436static int 1437apply_obj_config_map(struct bpf_map *map) 1438{ 1439 return bpf_map_config_foreach_key(map, 1440 apply_obj_config_map_for_key, 1441 NULL); 1442} 1443 1444static int 1445apply_obj_config_object(struct bpf_object *obj) 1446{ 1447 struct bpf_map *map; 1448 int err; 1449 1450 bpf_map__for_each(map, obj) { 1451 err = apply_obj_config_map(map); 1452 if (err) 1453 return err; 1454 } 1455 return 0; 1456} 1457 1458int bpf__apply_obj_config(void) 1459{ 1460 struct bpf_object *obj, *tmp; 1461 int err; 1462 1463 bpf_object__for_each_safe(obj, tmp) { 1464 err = apply_obj_config_object(obj); 1465 if (err) 1466 return err; 1467 } 1468 1469 return 0; 1470} 1471 1472#define bpf__for_each_map(pos, obj, objtmp) \ 1473 bpf_object__for_each_safe(obj, objtmp) \ 1474 bpf_map__for_each(pos, obj) 1475 1476#define bpf__for_each_stdout_map(pos, obj, objtmp) \ 1477 bpf__for_each_map(pos, obj, objtmp) \ 1478 if (bpf_map__get_name(pos) && \ 1479 (strcmp("__bpf_stdout__", \ 1480 bpf_map__get_name(pos)) == 0)) 1481 1482int bpf__setup_stdout(struct perf_evlist *evlist __maybe_unused) 1483{ 1484 struct bpf_map_priv *tmpl_priv = NULL; 1485 struct bpf_object *obj, *tmp; 1486 struct perf_evsel *evsel = NULL; 1487 struct bpf_map *map; 1488 int err; 1489 bool need_init = false; 1490 1491 bpf__for_each_stdout_map(map, obj, tmp) { 1492 struct bpf_map_priv *priv; 1493 1494 err = bpf_map__get_private(map, (void **)&priv); 1495 if (err) 1496 return -BPF_LOADER_ERRNO__INTERNAL; 1497 1498 /* 1499 * No need to check map type: type should have been 1500 * verified by kernel. 1501 */ 1502 if (!need_init && !priv) 1503 need_init = !priv; 1504 if (!tmpl_priv && priv) 1505 tmpl_priv = priv; 1506 } 1507 1508 if (!need_init) 1509 return 0; 1510 1511 if (!tmpl_priv) { 1512 err = parse_events(evlist, "bpf-output/no-inherit=1,name=__bpf_stdout__/", 1513 NULL); 1514 if (err) { 1515 pr_debug("ERROR: failed to create bpf-output event\n"); 1516 return -err; 1517 } 1518 1519 evsel = perf_evlist__last(evlist); 1520 } 1521 1522 bpf__for_each_stdout_map(map, obj, tmp) { 1523 struct bpf_map_priv *priv; 1524 1525 err = bpf_map__get_private(map, (void **)&priv); 1526 if (err) 1527 return -BPF_LOADER_ERRNO__INTERNAL; 1528 if (priv) 1529 continue; 1530 1531 if (tmpl_priv) { 1532 priv = bpf_map_priv__clone(tmpl_priv); 1533 if (!priv) 1534 return -ENOMEM; 1535 1536 err = bpf_map__set_private(map, priv, bpf_map_priv__clear); 1537 if (err) { 1538 bpf_map_priv__clear(map, priv); 1539 return err; 1540 } 1541 } else if (evsel) { 1542 struct bpf_map_op *op; 1543 1544 op = bpf_map__add_newop(map, NULL); 1545 if (IS_ERR(op)) 1546 return PTR_ERR(op); 1547 op->op_type = BPF_MAP_OP_SET_EVSEL; 1548 op->v.evsel = evsel; 1549 } 1550 } 1551 1552 return 0; 1553} 1554 1555#define ERRNO_OFFSET(e) ((e) - __BPF_LOADER_ERRNO__START) 1556#define ERRCODE_OFFSET(c) ERRNO_OFFSET(BPF_LOADER_ERRNO__##c) 1557#define NR_ERRNO (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START) 1558 1559static const char *bpf_loader_strerror_table[NR_ERRNO] = { 1560 [ERRCODE_OFFSET(CONFIG)] = "Invalid config string", 1561 [ERRCODE_OFFSET(GROUP)] = "Invalid group name", 1562 [ERRCODE_OFFSET(EVENTNAME)] = "No event name found in config string", 1563 [ERRCODE_OFFSET(INTERNAL)] = "BPF loader internal error", 1564 [ERRCODE_OFFSET(COMPILE)] = "Error when compiling BPF scriptlet", 1565 [ERRCODE_OFFSET(PROGCONF_TERM)] = "Invalid program config term in config string", 1566 [ERRCODE_OFFSET(PROLOGUE)] = "Failed to generate prologue", 1567 [ERRCODE_OFFSET(PROLOGUE2BIG)] = "Prologue too big for program", 1568 [ERRCODE_OFFSET(PROLOGUEOOB)] = "Offset out of bound for prologue", 1569 [ERRCODE_OFFSET(OBJCONF_OPT)] = "Invalid object config option", 1570 [ERRCODE_OFFSET(OBJCONF_CONF)] = "Config value not set (missing '=')", 1571 [ERRCODE_OFFSET(OBJCONF_MAP_OPT)] = "Invalid object map config option", 1572 [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST)] = "Target map doesn't exist", 1573 [ERRCODE_OFFSET(OBJCONF_MAP_VALUE)] = "Incorrect value type for map", 1574 [ERRCODE_OFFSET(OBJCONF_MAP_TYPE)] = "Incorrect map type", 1575 [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE)] = "Incorrect map key size", 1576 [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE)] = "Incorrect map value size", 1577 [ERRCODE_OFFSET(OBJCONF_MAP_NOEVT)] = "Event not found for map setting", 1578 [ERRCODE_OFFSET(OBJCONF_MAP_MAPSIZE)] = "Invalid map size for event setting", 1579 [ERRCODE_OFFSET(OBJCONF_MAP_EVTDIM)] = "Event dimension too large", 1580 [ERRCODE_OFFSET(OBJCONF_MAP_EVTINH)] = "Doesn't support inherit event", 1581 [ERRCODE_OFFSET(OBJCONF_MAP_EVTTYPE)] = "Wrong event type for map", 1582 [ERRCODE_OFFSET(OBJCONF_MAP_IDX2BIG)] = "Index too large", 1583}; 1584 1585static int 1586bpf_loader_strerror(int err, char *buf, size_t size) 1587{ 1588 char sbuf[STRERR_BUFSIZE]; 1589 const char *msg; 1590 1591 if (!buf || !size) 1592 return -1; 1593 1594 err = err > 0 ? err : -err; 1595 1596 if (err >= __LIBBPF_ERRNO__START) 1597 return libbpf_strerror(err, buf, size); 1598 1599 if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) { 1600 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)]; 1601 snprintf(buf, size, "%s", msg); 1602 buf[size - 1] = '\0'; 1603 return 0; 1604 } 1605 1606 if (err >= __BPF_LOADER_ERRNO__END) 1607 snprintf(buf, size, "Unknown bpf loader error %d", err); 1608 else 1609 snprintf(buf, size, "%s", 1610 strerror_r(err, sbuf, sizeof(sbuf))); 1611 1612 buf[size - 1] = '\0'; 1613 return -1; 1614} 1615 1616#define bpf__strerror_head(err, buf, size) \ 1617 char sbuf[STRERR_BUFSIZE], *emsg;\ 1618 if (!size)\ 1619 return 0;\ 1620 if (err < 0)\ 1621 err = -err;\ 1622 bpf_loader_strerror(err, sbuf, sizeof(sbuf));\ 1623 emsg = sbuf;\ 1624 switch (err) {\ 1625 default:\ 1626 scnprintf(buf, size, "%s", emsg);\ 1627 break; 1628 1629#define bpf__strerror_entry(val, fmt...)\ 1630 case val: {\ 1631 scnprintf(buf, size, fmt);\ 1632 break;\ 1633 } 1634 1635#define bpf__strerror_end(buf, size)\ 1636 }\ 1637 buf[size - 1] = '\0'; 1638 1639int bpf__strerror_prepare_load(const char *filename, bool source, 1640 int err, char *buf, size_t size) 1641{ 1642 size_t n; 1643 int ret; 1644 1645 n = snprintf(buf, size, "Failed to load %s%s: ", 1646 filename, source ? " from source" : ""); 1647 if (n >= size) { 1648 buf[size - 1] = '\0'; 1649 return 0; 1650 } 1651 buf += n; 1652 size -= n; 1653 1654 ret = bpf_loader_strerror(err, buf, size); 1655 buf[size - 1] = '\0'; 1656 return ret; 1657} 1658 1659int bpf__strerror_probe(struct bpf_object *obj __maybe_unused, 1660 int err, char *buf, size_t size) 1661{ 1662 bpf__strerror_head(err, buf, size); 1663 case BPF_LOADER_ERRNO__PROGCONF_TERM: { 1664 scnprintf(buf, size, "%s (add -v to see detail)", emsg); 1665 break; 1666 } 1667 bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'"); 1668 bpf__strerror_entry(EACCES, "You need to be root"); 1669 bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0"); 1670 bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file"); 1671 bpf__strerror_end(buf, size); 1672 return 0; 1673} 1674 1675int bpf__strerror_load(struct bpf_object *obj, 1676 int err, char *buf, size_t size) 1677{ 1678 bpf__strerror_head(err, buf, size); 1679 case LIBBPF_ERRNO__KVER: { 1680 unsigned int obj_kver = bpf_object__get_kversion(obj); 1681 unsigned int real_kver; 1682 1683 if (fetch_kernel_version(&real_kver, NULL, 0)) { 1684 scnprintf(buf, size, "Unable to fetch kernel version"); 1685 break; 1686 } 1687 1688 if (obj_kver != real_kver) { 1689 scnprintf(buf, size, 1690 "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")", 1691 KVER_PARAM(obj_kver), 1692 KVER_PARAM(real_kver)); 1693 break; 1694 } 1695 1696 scnprintf(buf, size, "Failed to load program for unknown reason"); 1697 break; 1698 } 1699 bpf__strerror_end(buf, size); 1700 return 0; 1701} 1702 1703int bpf__strerror_config_obj(struct bpf_object *obj __maybe_unused, 1704 struct parse_events_term *term __maybe_unused, 1705 struct perf_evlist *evlist __maybe_unused, 1706 int *error_pos __maybe_unused, int err, 1707 char *buf, size_t size) 1708{ 1709 bpf__strerror_head(err, buf, size); 1710 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE, 1711 "Can't use this config term with this map type"); 1712 bpf__strerror_end(buf, size); 1713 return 0; 1714} 1715 1716int bpf__strerror_apply_obj_config(int err, char *buf, size_t size) 1717{ 1718 bpf__strerror_head(err, buf, size); 1719 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM, 1720 "Cannot set event to BPF map in multi-thread tracing"); 1721 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH, 1722 "%s (Hint: use -i to turn off inherit)", emsg); 1723 bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE, 1724 "Can only put raw, hardware and BPF output event into a BPF map"); 1725 bpf__strerror_end(buf, size); 1726 return 0; 1727} 1728 1729int bpf__strerror_setup_stdout(struct perf_evlist *evlist __maybe_unused, 1730 int err, char *buf, size_t size) 1731{ 1732 bpf__strerror_head(err, buf, size); 1733 bpf__strerror_end(buf, size); 1734 return 0; 1735}