Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com * Copyright (c) 2016 Facebook * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io */#include <uapi/linux/btf.h>#include <linux/kernel.h>#include <linux/types.h>#include <linux/bpf.h>#include <linux/bpf_verifier.h>#include <linux/math64.h>#include <linux/string.h>
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
static bool bpf_verifier_log_attr_valid(u32 log_level, char __user *log_buf, u32 log_size){ /* ubuf and len_total should both be specified (or not) together */ if (!!log_buf != !!log_size) return false; /* log buf without log_level is meaningless */ if (log_buf && log_level == 0) return false; if (log_level & ~BPF_LOG_MASK) return false; if (log_size > UINT_MAX >> 2) return false; return true;}
int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level, char __user *log_buf, u32 log_size){ log->level = log_level; log->ubuf = log_buf; log->len_total = log_size;
/* log attributes have to be sane */ if (!bpf_verifier_log_attr_valid(log_level, log_buf, log_size)) return -EINVAL;
return 0;}
static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len){ /* add_len includes terminal \0, so no need for +1. */ u64 len = log->end_pos + add_len;
/* log->len_max could be larger than our current len due to * bpf_vlog_reset() calls, so we maintain the max of any length at any * previous point */ if (len > UINT_MAX) log->len_max = UINT_MAX; else if (len > log->len_max) log->len_max = len;}
void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args){ u64 cur_pos; u32 new_n, n;
n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
if (log->level == BPF_LOG_KERNEL) { bool newline = n > 0 && log->kbuf[n - 1] == '\n';
pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n"); return; }
n += 1; /* include terminating zero */ bpf_vlog_update_len_max(log, n);
if (log->level & BPF_LOG_FIXED) { /* check if we have at least something to put into user buf */ new_n = 0; if (log->end_pos < log->len_total) { new_n = min_t(u32, log->len_total - log->end_pos, n); log->kbuf[new_n - 1] = '\0'; }
cur_pos = log->end_pos; log->end_pos += n - 1; /* don't count terminating '\0' */
if (log->ubuf && new_n && copy_to_user(log->ubuf + cur_pos, log->kbuf, new_n)) goto fail; } else { u64 new_end, new_start; u32 buf_start, buf_end;
new_end = log->end_pos + n; if (new_end - log->start_pos >= log->len_total) new_start = new_end - log->len_total; else new_start = log->start_pos;
log->start_pos = new_start; log->end_pos = new_end - 1; /* don't count terminating '\0' */
if (!log->ubuf) return;
new_n = min(n, log->len_total); cur_pos = new_end - new_n; div_u64_rem(cur_pos, log->len_total, &buf_start); div_u64_rem(new_end, log->len_total, &buf_end); /* new_end and buf_end are exclusive indices, so if buf_end is * exactly zero, then it actually points right to the end of * ubuf and there is no wrap around */ if (buf_end == 0) buf_end = log->len_total;
/* if buf_start > buf_end, we wrapped around; * if buf_start == buf_end, then we fill ubuf completely; we * can't have buf_start == buf_end to mean that there is * nothing to write, because we always write at least * something, even if terminal '\0' */ if (buf_start < buf_end) { /* message fits within contiguous chunk of ubuf */ if (copy_to_user(log->ubuf + buf_start, log->kbuf + n - new_n, buf_end - buf_start)) goto fail; } else { /* message wraps around the end of ubuf, copy in two chunks */ if (copy_to_user(log->ubuf + buf_start, log->kbuf + n - new_n, log->len_total - buf_start)) goto fail; if (copy_to_user(log->ubuf, log->kbuf + n - buf_end, buf_end)) goto fail; } }
return;fail: log->ubuf = NULL;}
void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos){ char zero = 0; u32 pos;
if (WARN_ON_ONCE(new_pos > log->end_pos)) return;
if (!bpf_verifier_log_needed(log) || log->level == BPF_LOG_KERNEL) return;
/* if position to which we reset is beyond current log window, * then we didn't preserve any useful content and should adjust * start_pos to end up with an empty log (start_pos == end_pos) */ log->end_pos = new_pos; if (log->end_pos < log->start_pos) log->start_pos = log->end_pos;
if (!log->ubuf) return;
if (log->level & BPF_LOG_FIXED) pos = log->end_pos + 1; else div_u64_rem(new_pos, log->len_total, &pos);
if (pos < log->len_total && put_user(zero, log->ubuf + pos)) log->ubuf = NULL;}
static void bpf_vlog_reverse_kbuf(char *buf, int len){ int i, j;
for (i = 0, j = len - 1; i < j; i++, j--) swap(buf[i], buf[j]);}
static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end){ /* we split log->kbuf into two equal parts for both ends of array */ int n = sizeof(log->kbuf) / 2, nn; char *lbuf = log->kbuf, *rbuf = log->kbuf + n;
/* Read ubuf's section [start, end) two chunks at a time, from left * and right side; within each chunk, swap all the bytes; after that * reverse the order of lbuf and rbuf and write result back to ubuf. * This way we'll end up with swapped contents of specified * [start, end) ubuf segment. */ while (end - start > 1) { nn = min(n, (end - start ) / 2);
if (copy_from_user(lbuf, log->ubuf + start, nn)) return -EFAULT; if (copy_from_user(rbuf, log->ubuf + end - nn, nn)) return -EFAULT;
bpf_vlog_reverse_kbuf(lbuf, nn); bpf_vlog_reverse_kbuf(rbuf, nn);
/* we write lbuf to the right end of ubuf, while rbuf to the * left one to end up with properly reversed overall ubuf */ if (copy_to_user(log->ubuf + start, rbuf, nn)) return -EFAULT; if (copy_to_user(log->ubuf + end - nn, lbuf, nn)) return -EFAULT;
start += nn; end -= nn; }
return 0;}
int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual){ u32 sublen; int err;
*log_size_actual = 0; if (!log || log->level == 0 || log->level == BPF_LOG_KERNEL) return 0;
if (!log->ubuf) goto skip_log_rotate; /* If we never truncated log, there is nothing to move around. */ if (log->start_pos == 0) goto skip_log_rotate;
/* Otherwise we need to rotate log contents to make it start from the * buffer beginning and be a continuous zero-terminated string. Note * that if log->start_pos != 0 then we definitely filled up entire log * buffer with no gaps, and we just need to shift buffer contents to * the left by (log->start_pos % log->len_total) bytes. * * Unfortunately, user buffer could be huge and we don't want to * allocate temporary kernel memory of the same size just to shift * contents in a straightforward fashion. Instead, we'll be clever and * do in-place array rotation. This is a leetcode-style problem, which * could be solved by three rotations. * * Let's say we have log buffer that has to be shifted left by 7 bytes * (spaces and vertical bar is just for demonstrative purposes): * E F G H I J K | A B C D * * First, we reverse entire array: * D C B A | K J I H G F E * * Then we rotate first 4 bytes (DCBA) and separately last 7 bytes * (KJIHGFE), resulting in a properly rotated array: * A B C D | E F G H I J K * * We'll utilize log->kbuf to read user memory chunk by chunk, swap * bytes, and write them back. Doing it byte-by-byte would be * unnecessarily inefficient. Altogether we are going to read and * write each byte twice, for total 4 memory copies between kernel and * user space. */
/* length of the chopped off part that will be the beginning; * len(ABCD) in the example above */ div_u64_rem(log->start_pos, log->len_total, &sublen); sublen = log->len_total - sublen;
err = bpf_vlog_reverse_ubuf(log, 0, log->len_total); err = err ?: bpf_vlog_reverse_ubuf(log, 0, sublen); err = err ?: bpf_vlog_reverse_ubuf(log, sublen, log->len_total); if (err) log->ubuf = NULL;
skip_log_rotate: *log_size_actual = log->len_max;
/* properly initialized log has either both ubuf!=NULL and len_total>0 * or ubuf==NULL and len_total==0, so if this condition doesn't hold, * we got a fault somewhere along the way, so report it back */ if (!!log->ubuf != !!log->len_total) return -EFAULT;
/* did truncation actually happen? */ if (log->ubuf && log->len_max > log->len_total) return -ENOSPC;
return 0;}
/* log_level controls verbosity level of eBPF verifier. * bpf_verifier_log_write() is used to dump the verification trace to the log, * so the user can figure out what's wrong with the program */__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, const char *fmt, ...){ va_list args;
if (!bpf_verifier_log_needed(&env->log)) return;
va_start(args, fmt); bpf_verifier_vlog(&env->log, fmt, args); va_end(args);}EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
__printf(2, 3) void bpf_log(struct bpf_verifier_log *log, const char *fmt, ...){ va_list args;
if (!bpf_verifier_log_needed(log)) return;
va_start(args, fmt); bpf_verifier_vlog(log, fmt, args); va_end(args);}EXPORT_SYMBOL_GPL(bpf_log);
static const char *ltrim(const char *s){ while (isspace(*s)) s++;
return s;}
__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...){ const struct bpf_line_info *linfo, *prev_linfo; const struct btf *btf; const char *s, *fname;
if (!bpf_verifier_log_needed(&env->log)) return;
prev_linfo = env->prev_linfo; linfo = bpf_find_linfo(env->prog, insn_off); if (!linfo || linfo == prev_linfo) return;
/* It often happens that two separate linfo records point to the same * source code line, but have differing column numbers. Given verifier * log doesn't emit column information, from user perspective we just * end up emitting the same source code line twice unnecessarily. * So instead check that previous and current linfo record point to * the same file (file_name_offs match) and the same line number, and * avoid emitting duplicated source code line in such case. */ if (prev_linfo && linfo->file_name_off == prev_linfo->file_name_off && BPF_LINE_INFO_LINE_NUM(linfo->line_col) == BPF_LINE_INFO_LINE_NUM(prev_linfo->line_col)) return;
if (prefix_fmt) { va_list args;
va_start(args, prefix_fmt); bpf_verifier_vlog(&env->log, prefix_fmt, args); va_end(args); }
btf = env->prog->aux->btf; s = ltrim(btf_name_by_offset(btf, linfo->line_off)); verbose(env, "%s", s); /* source code line */
s = btf_name_by_offset(btf, linfo->file_name_off); /* leave only file name */ fname = strrchr(s, '/'); fname = fname ? fname + 1 : s; verbose(env, " @ %s:%u\n", fname, BPF_LINE_INFO_LINE_NUM(linfo->line_col));
env->prev_linfo = linfo;}
static const char *btf_type_name(const struct btf *btf, u32 id){ return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);}
/* string representation of 'enum bpf_reg_type' * * Note that reg_type_str() can not appear more than once in a single verbose() * statement. */const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type){ char postfix[16] = {0}, prefix[64] = {0}; static const char * const str[] = { [NOT_INIT] = "?", [SCALAR_VALUE] = "scalar", [PTR_TO_CTX] = "ctx", [CONST_PTR_TO_MAP] = "map_ptr", [PTR_TO_MAP_VALUE] = "map_value", [PTR_TO_STACK] = "fp", [PTR_TO_PACKET] = "pkt", [PTR_TO_PACKET_META] = "pkt_meta", [PTR_TO_PACKET_END] = "pkt_end", [PTR_TO_FLOW_KEYS] = "flow_keys", [PTR_TO_SOCKET] = "sock", [PTR_TO_SOCK_COMMON] = "sock_common", [PTR_TO_TCP_SOCK] = "tcp_sock", [PTR_TO_TP_BUFFER] = "tp_buffer", [PTR_TO_XDP_SOCK] = "xdp_sock", [PTR_TO_BTF_ID] = "ptr_", [PTR_TO_MEM] = "mem", [PTR_TO_ARENA] = "arena", [PTR_TO_BUF] = "buf", [PTR_TO_FUNC] = "func", [PTR_TO_INSN] = "insn", [PTR_TO_MAP_KEY] = "map_key", [CONST_PTR_TO_DYNPTR] = "dynptr_ptr", };
if (type & PTR_MAYBE_NULL) { if (base_type(type) == PTR_TO_BTF_ID) strscpy(postfix, "or_null_"); else strscpy(postfix, "_or_null"); }
snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s", type & MEM_RDONLY ? "rdonly_" : "", type & MEM_RINGBUF ? "ringbuf_" : "", type & MEM_USER ? "user_" : "", type & MEM_PERCPU ? "percpu_" : "", type & MEM_RCU ? "rcu_" : "", type & PTR_UNTRUSTED ? "untrusted_" : "", type & PTR_TRUSTED ? "trusted_" : "" );
snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s", prefix, str[base_type(type)], postfix); return env->tmp_str_buf;}
const char *dynptr_type_str(enum bpf_dynptr_type type){ switch (type) { case BPF_DYNPTR_TYPE_LOCAL: return "local"; case BPF_DYNPTR_TYPE_RINGBUF: return "ringbuf"; case BPF_DYNPTR_TYPE_SKB: return "skb"; case BPF_DYNPTR_TYPE_XDP: return "xdp"; case BPF_DYNPTR_TYPE_SKB_META: return "skb_meta"; case BPF_DYNPTR_TYPE_FILE: return "file"; case BPF_DYNPTR_TYPE_INVALID: return "<invalid>"; default: WARN_ONCE(1, "unknown dynptr type %d\n", type); return "<unknown>"; }}
const char *iter_type_str(const struct btf *btf, u32 btf_id){ if (!btf || btf_id == 0) return "<invalid>";
/* we already validated that type is valid and has conforming name */ return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1;}
const char *iter_state_str(enum bpf_iter_state state){ switch (state) { case BPF_ITER_STATE_ACTIVE: return "active"; case BPF_ITER_STATE_DRAINED: return "drained"; case BPF_ITER_STATE_INVALID: return "<invalid>"; default: WARN_ONCE(1, "unknown iter state %d\n", state); return "<unknown>"; }}
static char slot_type_char[] = { [STACK_INVALID] = '?', [STACK_SPILL] = 'r', [STACK_MISC] = 'm', [STACK_ZERO] = '0', [STACK_DYNPTR] = 'd', [STACK_ITER] = 'i', [STACK_IRQ_FLAG] = 'f', [STACK_POISON] = 'p',};
#define UNUM_MAX_DECIMAL U16_MAX#define SNUM_MAX_DECIMAL S16_MAX#define SNUM_MIN_DECIMAL S16_MIN
static bool is_unum_decimal(u64 num){ return num <= UNUM_MAX_DECIMAL;}
static bool is_snum_decimal(s64 num){ return num >= SNUM_MIN_DECIMAL && num <= SNUM_MAX_DECIMAL;}
static void verbose_unum(struct bpf_verifier_env *env, u64 num){ if (is_unum_decimal(num)) verbose(env, "%llu", num); else verbose(env, "%#llx", num);}
static void verbose_snum(struct bpf_verifier_env *env, s64 num){ if (is_snum_decimal(num)) verbose(env, "%lld", num); else verbose(env, "%#llx", num);}
int tnum_strn(char *str, size_t size, struct tnum a){ /* print as a constant, if tnum is fully known */ if (a.mask == 0) { if (is_unum_decimal(a.value)) return snprintf(str, size, "%llu", a.value); if (is_snum_decimal(a.value)) return snprintf(str, size, "%lld", a.value); else return snprintf(str, size, "%#llx", a.value); } return snprintf(str, size, "(%#llx; %#llx)", a.value, a.mask);}EXPORT_SYMBOL_GPL(tnum_strn);
static void print_scalar_ranges(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const char **sep){ /* For signed ranges, we want to unify 64-bit and 32-bit values in the * output as much as possible, but there is a bit of a complication. * If we choose to print values as decimals, this is natural to do, * because negative 64-bit and 32-bit values >= -S32_MIN have the same * representation due to sign extension. But if we choose to print * them in hex format (see is_snum_decimal()), then sign extension is * misleading. * E.g., smin=-2 and smin32=-2 are exactly the same in decimal, but in * hex they will be smin=0xfffffffffffffffe and smin32=0xfffffffe, two * very different numbers. * So we avoid sign extension if we choose to print values in hex. */ struct { const char *name; u64 val; bool omit; } minmaxs[] = { {"smin", reg_smin(reg), reg_smin(reg) == S64_MIN}, {"smax", reg_smax(reg), reg_smax(reg) == S64_MAX}, {"umin", reg_umin(reg), reg_umin(reg) == 0}, {"umax", reg_umax(reg), reg_umax(reg) == U64_MAX}, {"smin32", is_snum_decimal((s64)reg_s32_min(reg)) ? (s64)reg_s32_min(reg) : (u32)reg_s32_min(reg), reg_s32_min(reg) == S32_MIN}, {"smax32", is_snum_decimal((s64)reg_s32_max(reg)) ? (s64)reg_s32_max(reg) : (u32)reg_s32_max(reg), reg_s32_max(reg) == S32_MAX}, {"umin32", reg_u32_min(reg), reg_u32_min(reg) == 0}, {"umax32", reg_u32_max(reg), reg_u32_max(reg) == U32_MAX}, }, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)]; bool neg1, neg2;
for (m1 = &minmaxs[0]; m1 < mend; m1++) { if (m1->omit) continue;
neg1 = m1->name[0] == 's' && (s64)m1->val < 0;
verbose(env, "%s%s=", *sep, m1->name); *sep = ",";
for (m2 = m1 + 2; m2 < mend; m2 += 2) { if (m2->omit || m2->val != m1->val) continue; /* don't mix negatives with positives */ neg2 = m2->name[0] == 's' && (s64)m2->val < 0; if (neg2 != neg1) continue; m2->omit = true; verbose(env, "%s=", m2->name); }
if (m1->name[0] == 's') verbose_snum(env, m1->val); else verbose_unum(env, m1->val); }}
static bool type_is_map_ptr(enum bpf_reg_type t) { switch (base_type(t)) { case CONST_PTR_TO_MAP: case PTR_TO_MAP_KEY: case PTR_TO_MAP_VALUE: return true; default: return false; }}
/* * _a stands for append, was shortened to avoid multiline statements below. * This macro is used to output a comma separated list of attributes. */#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; })
static void print_reg_state(struct bpf_verifier_env *env, const struct bpf_func_state *state, const struct bpf_reg_state *reg){ enum bpf_reg_type t; const char *sep = "";
t = reg->type; if (t == SCALAR_VALUE && reg->precise) verbose(env, "P"); if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) { verbose_snum(env, reg->var_off.value); return; }
verbose(env, "%s", reg_type_str(env, t)); if (t == PTR_TO_ARENA) return; if (t == PTR_TO_STACK) { if (state->frameno != reg->frameno) verbose(env, "[%d]", reg->frameno); if (tnum_is_const(reg->var_off)) { verbose_snum(env, reg->var_off.value + reg->delta); return; } } if (base_type(t) == PTR_TO_BTF_ID) verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id)); verbose(env, "("); if (reg->id) verbose_a("id=%d", reg->id & ~BPF_ADD_CONST); if (reg->id & BPF_ADD_CONST) verbose(env, "%+d", reg->delta); if (reg->parent_id) verbose_a("parent_id=%d", reg->parent_id); if (type_is_non_owning_ref(reg->type)) verbose_a("%s", "non_own_ref"); if (type_is_map_ptr(t)) { if (reg->map_ptr->name[0]) verbose_a("map=%s", reg->map_ptr->name); verbose_a("ks=%d,vs=%d", reg->map_ptr->key_size, reg->map_ptr->value_size); } if (t != SCALAR_VALUE && reg->delta) { verbose_a("off="); verbose_snum(env, reg->delta); } if (type_is_pkt_pointer(t)) { verbose_a("r="); verbose_unum(env, reg->range); } if (base_type(t) == PTR_TO_MEM) { verbose_a("sz="); verbose_unum(env, reg->mem_size); } if (t == CONST_PTR_TO_DYNPTR) verbose_a("type=%s", dynptr_type_str(reg->dynptr.type)); if (tnum_is_const(reg->var_off)) { /* a pointer register with fixed offset */ if (reg->var_off.value) { verbose_a("imm="); verbose_snum(env, reg->var_off.value); } } else { print_scalar_ranges(env, reg, &sep); if (!tnum_is_unknown(reg->var_off)) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose_a("var_off=%s", tn_buf); } } verbose(env, ")");}
void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, u32 frameno, bool print_all){ const struct bpf_func_state *state = vstate->frame[frameno]; const struct bpf_reg_state *reg; int i;
if (state->frameno) verbose(env, " frame%d:", state->frameno); for (i = 0; i < MAX_BPF_REG; i++) { reg = &state->regs[i]; if (reg->type == NOT_INIT) continue; if (!print_all && !reg_scratched(env, i)) continue; verbose(env, " R%d", i); verbose(env, "="); print_reg_state(env, state, reg); } for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { char types_buf[BPF_REG_SIZE + 1]; const char *sep = ""; bool valid = false; u8 slot_type; int j;
if (!print_all && !stack_slot_scratched(env, i)) continue;
for (j = 0; j < BPF_REG_SIZE; j++) { slot_type = state->stack[i].slot_type[j]; if (slot_type != STACK_INVALID && slot_type != STACK_POISON) valid = true; types_buf[j] = slot_type_char[slot_type]; } types_buf[BPF_REG_SIZE] = 0; if (!valid) continue;
reg = &state->stack[i].spilled_ptr; switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { case STACK_SPILL: /* print MISC/ZERO/INVALID slots above subreg spill */ for (j = 0; j < BPF_REG_SIZE; j++) if (state->stack[i].slot_type[j] == STACK_SPILL) break; types_buf[j] = '\0';
verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf); print_reg_state(env, state, reg); break; case STACK_DYNPTR: /* skip to main dynptr slot */ i += BPF_DYNPTR_NR_SLOTS - 1; reg = &state->stack[i].spilled_ptr;
verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type)); if (reg->id) verbose_a("id=%d", reg->id); if (reg->parent_id) verbose_a("parent_id=%d", reg->parent_id); verbose(env, ")"); break; case STACK_ITER: /* only main slot has id set; skip others */ if (!reg->id) continue;
verbose(env, " fp%d=iter_%s(id=%d,state=%s,depth=%u)", (-i - 1) * BPF_REG_SIZE, iter_type_str(reg->iter.btf, reg->iter.btf_id), reg->id, iter_state_str(reg->iter.state), reg->iter.depth); break; case STACK_MISC: case STACK_ZERO: default: verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf); break; } } if (vstate->acquired_refs && vstate->refs[0].id) { verbose(env, " refs=%d", vstate->refs[0].id); for (i = 1; i < vstate->acquired_refs; i++) if (vstate->refs[i].id) verbose(env, ",%d", vstate->refs[i].id); } if (state->in_callback_fn) verbose(env, " cb"); if (state->in_async_callback_fn) verbose(env, " async_cb"); verbose(env, "\n"); if (!print_all) mark_verifier_state_clean(env);}
u32 bpf_vlog_alignment(u32 pos){ return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), BPF_LOG_MIN_ALIGNMENT) - pos - 1;}
void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, u32 frameno){ if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) { /* remove new line character */ bpf_vlog_reset(&env->log, env->prev_log_pos - 1); verbose(env, "%*c;", bpf_vlog_alignment(env->prev_insn_print_pos), ' '); } else { verbose(env, "%d:", env->insn_idx); } print_verifier_state(env, vstate, frameno, false);}
int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 log_level, u32 offsetof_log_true_size, bpfptr_t uattr, struct bpf_common_attr *common, bpfptr_t uattr_common, u32 size_common){ char __user *ubuf_common = u64_to_user_ptr(common->log_buf); char __user *ubuf = u64_to_user_ptr(log_buf);
if (!bpf_verifier_log_attr_valid(common->log_level, ubuf_common, common->log_size) || !bpf_verifier_log_attr_valid(log_level, ubuf, log_size)) return -EINVAL;
if (ubuf && ubuf_common && (ubuf != ubuf_common || log_size != common->log_size || log_level != common->log_level)) return -EINVAL;
memset(log, 0, sizeof(*log)); log->ubuf = ubuf; log->size = log_size; log->level = log_level; log->offsetof_true_size = offsetof_log_true_size; log->uattr = uattr;
if (!ubuf && ubuf_common) { log->ubuf = ubuf_common; log->size = common->log_size; log->level = common->log_level; log->uattr = uattr_common; log->offsetof_true_size = 0; if (size_common >= offsetofend(struct bpf_common_attr, log_true_size)) log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size); } return 0;}
struct bpf_verifier_log *bpf_log_attr_create_vlog(struct bpf_log_attr *attr_log, struct bpf_common_attr *common, bpfptr_t uattr, u32 size){ struct bpf_verifier_log *log; int err;
memset(attr_log, 0, sizeof(*attr_log)); attr_log->uattr = uattr; if (size >= offsetofend(struct bpf_common_attr, log_true_size)) attr_log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size);
if (!size) return NULL;
log = kzalloc_obj(*log, GFP_KERNEL); if (!log) return ERR_PTR(-ENOMEM);
err = bpf_vlog_init(log, common->log_level, u64_to_user_ptr(common->log_buf), common->log_size); if (err) { kfree(log); return ERR_PTR(err); }
return log;}
int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *log){ u32 log_true_size; int err;
err = bpf_vlog_finalize(log, &log_true_size);
if (attr->offsetof_true_size && copy_to_bpfptr_offset(attr->uattr, attr->offsetof_true_size, &log_true_size, sizeof(log_true_size))) return -EFAULT;
return err;}