Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */#include <linux/bpf.h>#include <linux/btf.h>#include <linux/bpf_verifier.h>#include <linux/filter.h>#include <linux/vmalloc.h>#include <linux/bsearch.h>#include <linux/sort.h>#include <linux/perf_event.h>#include <net/xdp.h>#include "disasm.h"
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
static bool is_cmpxchg_insn(const struct bpf_insn *insn){ return BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_CMPXCHG;}
/* Return the regno defined by the insn, or -1. */static int insn_def_regno(const struct bpf_insn *insn){ switch (BPF_CLASS(insn->code)) { case BPF_JMP: case BPF_JMP32: case BPF_ST: return -1; case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC || BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { if (insn->imm == BPF_CMPXCHG) return BPF_REG_0; else if (insn->imm == BPF_LOAD_ACQ) return insn->dst_reg; else if (insn->imm & BPF_FETCH) return insn->src_reg; } return -1; default: return insn->dst_reg; }}
/* Return TRUE if INSN has defined any 32-bit value explicitly. */static bool insn_has_def32(struct bpf_insn *insn){ int dst_reg = insn_def_regno(insn);
if (dst_reg == -1) return false;
return !bpf_is_reg64(insn, dst_reg, NULL, DST_OP);}
static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b){ const struct bpf_kfunc_desc *d0 = a; const struct bpf_kfunc_desc *d1 = b;
if (d0->imm != d1->imm) return d0->imm < d1->imm ? -1 : 1; if (d0->offset != d1->offset) return d0->offset < d1->offset ? -1 : 1; return 0;}
const struct btf_func_model *bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn){ const struct bpf_kfunc_desc desc = { .imm = insn->imm, .offset = insn->off, }; const struct bpf_kfunc_desc *res; struct bpf_kfunc_desc_tab *tab;
tab = prog->aux->kfunc_tab; res = bsearch(&desc, tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off);
return res ? &res->func_model : NULL;}
static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc){ unsigned long call_imm;
if (bpf_jit_supports_far_kfunc_call()) { call_imm = desc->func_id; } else { call_imm = BPF_CALL_IMM(desc->addr); /* Check whether the relative offset overflows desc->imm */ if ((unsigned long)(s32)call_imm != call_imm) { verbose(env, "address of kernel func_id %u is out of range\n", desc->func_id); return -EINVAL; } } desc->imm = call_imm; return 0;}
static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env){ struct bpf_kfunc_desc_tab *tab; int i, err;
tab = env->prog->aux->kfunc_tab; if (!tab) return 0;
for (i = 0; i < tab->nr_descs; i++) { err = set_kfunc_desc_imm(env, &tab->descs[i]); if (err) return err; }
sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off, NULL); return 0;}
static int add_kfunc_in_insns(struct bpf_verifier_env *env, struct bpf_insn *insn, int cnt){ int i, ret;
for (i = 0; i < cnt; i++, insn++) { if (bpf_pseudo_kfunc_call(insn)) { ret = bpf_add_kfunc_call(env, insn->imm, insn->off); if (ret < 0) return ret; } } return 0;}
#ifndef CONFIG_BPF_JIT_ALWAYS_ONstatic int get_callee_stack_depth(struct bpf_verifier_env *env, const struct bpf_insn *insn, int idx){ int start = idx + insn->imm + 1, subprog;
subprog = bpf_find_subprog(env, start); if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start)) return -EFAULT; return env->subprog_info[subprog].stack_depth;}#endif
/* single env->prog->insni[off] instruction was replaced with the range * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying * [0, off) and [off, end) to new locations, so the patched range stays zero */static void adjust_insn_aux_data(struct bpf_verifier_env *env, struct bpf_prog *new_prog, u32 off, u32 cnt){ struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; u32 old_seen = data[off].seen; u32 prog_len; int i;
/* aux info at OFF always needs adjustment, no matter fast path * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the * original insn at old prog. */ data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);
if (cnt == 1) return; prog_len = new_prog->len;
memmove(data + off + cnt - 1, data + off, sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1)); for (i = off; i < off + cnt - 1; i++) { /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = insn_has_def32(insn + i); }
/* * The indirect_target flag of the original instruction was moved to the last of the * new instructions by the above memmove and memset, but the indirect jump target is * actually the first instruction, so move it back. This also matches with the behavior * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new * instruction. */ if (data[off + cnt - 1].indirect_target) { data[off].indirect_target = 1; data[off + cnt - 1].indirect_target = 0; }}
static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len){ int i;
if (len == 1) return; /* NOTE: fake 'exit' subprog should be updated as well. */ for (i = 0; i <= env->subprog_cnt; i++) { if (env->subprog_info[i].start <= off) continue; env->subprog_info[i].start += len - 1; }}
static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len){ int i;
if (len == 1) return;
for (i = 0; i < env->insn_array_map_cnt; i++) bpf_insn_array_adjust(env->insn_array_maps[i], off, len);}
static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len){ int i;
for (i = 0; i < env->insn_array_map_cnt; i++) bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len);}
static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len){ struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; int i, sz = prog->aux->size_poke_tab; struct bpf_jit_poke_descriptor *desc;
for (i = 0; i < sz; i++) { desc = &tab[i]; if (desc->insn_idx <= off) continue; desc->insn_idx += len - 1; }}
struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len){ struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL;
if (len > 1) { new_data = vrealloc(env->insn_aux_data, array_size(env->prog->len + len - 1, sizeof(struct bpf_insn_aux_data)), GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (!new_data) return NULL;
env->insn_aux_data = new_data; }
new_prog = bpf_patch_insn_single(env->prog, off, patch, len); if (IS_ERR(new_prog)) { if (PTR_ERR(new_prog) == -ERANGE) verbose(env, "insn %d cannot be patched due to 16-bit range\n", env->insn_aux_data[off].orig_idx); return NULL; } adjust_insn_aux_data(env, new_prog, off, len); adjust_subprog_starts(env, off, len); adjust_insn_arrays(env, off, len); adjust_poke_descs(new_prog, off, len); return new_prog;}
/* * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the * jump offset by 'delta'. */static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta){ struct bpf_insn *insn = prog->insnsi; u32 insn_cnt = prog->len, i; s32 imm; s16 off;
for (i = 0; i < insn_cnt; i++, insn++) { u8 code = insn->code;
if (tgt_idx <= i && i < tgt_idx + delta) continue;
if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT) continue;
if (insn->code == (BPF_JMP32 | BPF_JA)) { if (i + 1 + insn->imm != tgt_idx) continue; if (check_add_overflow(insn->imm, delta, &imm)) return -ERANGE; insn->imm = imm; } else { if (i + 1 + insn->off != tgt_idx) continue; if (check_add_overflow(insn->off, delta, &off)) return -ERANGE; insn->off = off; } } return 0;}
static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, u32 off, u32 cnt){ int i, j;
/* find first prog starting at or after off (first to remove) */ for (i = 0; i < env->subprog_cnt; i++) if (env->subprog_info[i].start >= off) break; /* find first prog starting at or after off + cnt (first to stay) */ for (j = i; j < env->subprog_cnt; j++) if (env->subprog_info[j].start >= off + cnt) break; /* if j doesn't start exactly at off + cnt, we are just removing * the front of previous prog */ if (env->subprog_info[j].start != off + cnt) j--;
if (j > i) { struct bpf_prog_aux *aux = env->prog->aux; int move;
/* move fake 'exit' subprog as well */ move = env->subprog_cnt + 1 - j;
memmove(env->subprog_info + i, env->subprog_info + j, sizeof(*env->subprog_info) * move); env->subprog_cnt -= j - i;
/* remove func_info */ if (aux->func_info) { move = aux->func_info_cnt - j;
memmove(aux->func_info + i, aux->func_info + j, sizeof(*aux->func_info) * move); aux->func_info_cnt -= j - i; /* func_info->insn_off is set after all code rewrites, * in adjust_btf_func() - no need to adjust */ } } else { /* convert i from "first prog to remove" to "first to adjust" */ if (env->subprog_info[i].start == off) i++; }
/* update fake 'exit' subprog as well */ for (; i <= env->subprog_cnt; i++) env->subprog_info[i].start -= cnt;
return 0;}
static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, u32 cnt){ struct bpf_prog *prog = env->prog; u32 i, l_off, l_cnt, nr_linfo; struct bpf_line_info *linfo;
nr_linfo = prog->aux->nr_linfo; if (!nr_linfo) return 0;
linfo = prog->aux->linfo;
/* find first line info to remove, count lines to be removed */ for (i = 0; i < nr_linfo; i++) if (linfo[i].insn_off >= off) break;
l_off = i; l_cnt = 0; for (; i < nr_linfo; i++) if (linfo[i].insn_off < off + cnt) l_cnt++; else break;
/* First live insn doesn't match first live linfo, it needs to "inherit" * last removed linfo. prog is already modified, so prog->len == off * means no live instructions after (tail of the program was removed). */ if (prog->len != off && l_cnt && (i == nr_linfo || linfo[i].insn_off != off + cnt)) { l_cnt--; linfo[--i].insn_off = off + cnt; }
/* remove the line info which refer to the removed instructions */ if (l_cnt) { memmove(linfo + l_off, linfo + i, sizeof(*linfo) * (nr_linfo - i));
prog->aux->nr_linfo -= l_cnt; nr_linfo = prog->aux->nr_linfo; }
/* pull all linfo[i].insn_off >= off + cnt in by cnt */ for (i = l_off; i < nr_linfo; i++) linfo[i].insn_off -= cnt;
/* fix up all subprogs (incl. 'exit') which start >= off */ for (i = 0; i <= env->subprog_cnt; i++) if (env->subprog_info[i].linfo_idx > l_off) { /* program may have started in the removed region but * may not be fully removed */ if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) env->subprog_info[i].linfo_idx -= l_cnt; else env->subprog_info[i].linfo_idx = l_off; }
return 0;}
/* * Clean up dynamically allocated fields of aux data for instructions [start, ...] */void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len){ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; struct bpf_insn *insns = env->prog->insnsi; int end = start + len; int i;
for (i = start; i < end; i++) { if (aux_data[i].jt) { kvfree(aux_data[i].jt); aux_data[i].jt = NULL; }
if (bpf_is_ldimm64(&insns[i])) i++; }}
static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt){ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; unsigned int orig_prog_len = env->prog->len; int err;
if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt);
/* Should be called before bpf_remove_insns, as it uses prog->insnsi */ bpf_clear_insn_aux_data(env, off, cnt);
err = bpf_remove_insns(env->prog, off, cnt); if (err) return err;
err = adjust_subprog_starts_after_remove(env, off, cnt); if (err) return err;
err = bpf_adj_linfo_after_remove(env, off, cnt); if (err) return err;
adjust_insn_arrays_after_remove(env, off, cnt);
memmove(aux_data + off, aux_data + off + cnt, sizeof(*aux_data) * (orig_prog_len - off - cnt));
return 0;}
static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0);static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0);
bool bpf_insn_is_cond_jump(u8 code){ u8 op;
op = BPF_OP(code); if (BPF_CLASS(code) == BPF_JMP32) return op != BPF_JA;
if (BPF_CLASS(code) != BPF_JMP) return false;
return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;}
void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env){ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); struct bpf_insn *insn = env->prog->insnsi; const int insn_cnt = env->prog->len; int i;
for (i = 0; i < insn_cnt; i++, insn++) { if (!bpf_insn_is_cond_jump(insn->code)) continue;
if (!aux_data[i + 1].seen) ja.off = insn->off; else if (!aux_data[i + 1 + insn->off].seen) ja.off = 0; else continue;
if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_replace_insn(env, i, &ja);
memcpy(insn, &ja, sizeof(ja)); }}
int bpf_opt_remove_dead_code(struct bpf_verifier_env *env){ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; int insn_cnt = env->prog->len; int i, err;
for (i = 0; i < insn_cnt; i++) { int j;
j = 0; while (i + j < insn_cnt && !aux_data[i + j].seen) j++; if (!j) continue;
err = verifier_remove_insns(env, i, j); if (err) return err; insn_cnt = env->prog->len; }
return 0;}
int bpf_opt_remove_nops(struct bpf_verifier_env *env){ struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; bool is_may_goto_0, is_ja; int i, err;
for (i = 0; i < insn_cnt; i++) { is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP));
if (!is_may_goto_0 && !is_ja) continue;
err = verifier_remove_insns(env, i, 1); if (err) return err; insn_cnt--; /* Go back one insn to catch may_goto +1; may_goto +0 sequence */ i -= (is_may_goto_0 && i > 0) ? 2 : 1; }
return 0;}
int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, const union bpf_attr *attr){ struct bpf_insn *patch; /* use env->insn_buf as two independent buffers */ struct bpf_insn *zext_patch = env->insn_buf; struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2]; struct bpf_insn_aux_data *aux = env->insn_aux_data; int i, patch_len, delta = 0, len = env->prog->len; struct bpf_insn *insns = env->prog->insnsi; struct bpf_prog *new_prog; bool rnd_hi32;
rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; zext_patch[1] = BPF_ZEXT_REG(0); rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); for (i = 0; i < len; i++) { int adj_idx = i + delta; struct bpf_insn insn; int load_reg;
insn = insns[adj_idx]; load_reg = insn_def_regno(&insn); if (!aux[adj_idx].zext_dst) { u8 code, class; u32 imm_rnd;
if (!rnd_hi32) continue;
code = insn.code; class = BPF_CLASS(code); if (load_reg == -1) continue;
/* NOTE: arg "reg" (the fourth one) is only used for * BPF_STX + SRC_OP, so it is safe to pass NULL * here. */ if (bpf_is_reg64(&insn, load_reg, NULL, DST_OP)) { if (class == BPF_LD && BPF_MODE(code) == BPF_IMM) i++; continue; }
/* ctx load could be transformed into wider load. */ if (class == BPF_LDX && aux[adj_idx].ptr_type == PTR_TO_CTX) continue;
imm_rnd = get_random_u32(); rnd_hi32_patch[0] = insn; rnd_hi32_patch[1].imm = imm_rnd; rnd_hi32_patch[3].dst_reg = load_reg; patch = rnd_hi32_patch; patch_len = 4; goto apply_patch_buffer; }
/* Add in an zero-extend instruction if a) the JIT has requested * it or b) it's a CMPXCHG. * * The latter is because: BPF_CMPXCHG always loads a value into * R0, therefore always zero-extends. However some archs' * equivalent instruction only does this load when the * comparison is successful. This detail of CMPXCHG is * orthogonal to the general zero-extension behaviour of the * CPU, so it's treated independently of bpf_jit_needs_zext. */ if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) continue;
/* Zero-extension is done by the caller. */ if (bpf_pseudo_kfunc_call(&insn)) continue;
if (verifier_bug_if(load_reg == -1, env, "zext_dst is set, but no reg is defined")) return -EFAULT;
zext_patch[0] = insn; zext_patch[1].dst_reg = load_reg; zext_patch[1].src_reg = load_reg; patch = zext_patch; patch_len = 2;apply_patch_buffer: new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); if (!new_prog) return -ENOMEM; env->prog = new_prog; insns = new_prog->insnsi; aux = env->insn_aux_data; delta += patch_len - 1; }
return 0;}
/* convert load instructions that access fields of a context type into a * sequence of instructions that access fields of the underlying structure: * struct __sk_buff -> struct sk_buff * struct bpf_sock_ops -> struct sock */int bpf_convert_ctx_accesses(struct bpf_verifier_env *env){ struct bpf_subprog_info *subprogs = env->subprog_info; const struct bpf_verifier_ops *ops = env->ops; int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0; const int insn_cnt = env->prog->len; struct bpf_insn *epilogue_buf = env->epilogue_buf; struct bpf_insn *insn_buf = env->insn_buf; struct bpf_insn *insn; u32 target_size, size_default, off; struct bpf_prog *new_prog; enum bpf_access_type type; bool is_narrower_load; int epilogue_idx = 0;
if (ops->gen_epilogue) { epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, -(subprogs[0].stack_depth + 8)); if (epilogue_cnt >= INSN_BUF_SIZE) { verifier_bug(env, "epilogue is too long"); return -EFAULT; } else if (epilogue_cnt) { /* Save the ARG_PTR_TO_CTX for the epilogue to use */ cnt = 0; subprogs[0].stack_depth += 8; insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, -subprogs[0].stack_depth); insn_buf[cnt++] = env->prog->insnsi[0]; new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); if (!new_prog) return -ENOMEM; env->prog = new_prog; delta += cnt - 1;
ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1); if (ret < 0) return ret; } }
if (ops->gen_prologue || env->seen_direct_write) { if (!ops->gen_prologue) { verifier_bug(env, "gen_prologue is null"); return -EFAULT; } cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, env->prog); if (cnt >= INSN_BUF_SIZE) { verifier_bug(env, "prologue is too long"); return -EFAULT; } else if (cnt) { new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); if (!new_prog) return -ENOMEM;
env->prog = new_prog; delta += cnt - 1;
ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); if (ret < 0) return ret; } }
if (delta) WARN_ON(adjust_jmp_off(env->prog, 0, delta));
if (bpf_prog_is_offloaded(env->prog->aux)) return 0;
insn = env->prog->insnsi + delta;
for (i = 0; i < insn_cnt; i++, insn++) { bpf_convert_ctx_access_t convert_ctx_access; u8 mode;
if (env->insn_aux_data[i + delta].nospec) { WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state); struct bpf_insn *patch = insn_buf;
*patch++ = BPF_ST_NOSPEC(); *patch++ = *insn; cnt = patch - insn_buf; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = new_prog; insn = new_prog->insnsi + i + delta; /* This can not be easily merged with the * nospec_result-case, because an insn may require a * nospec before and after itself. Therefore also do not * 'continue' here but potentially apply further * patching to insn. *insn should equal patch[1] now. */ }
if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || insn->code == (BPF_LDX | BPF_MEM | BPF_H) || insn->code == (BPF_LDX | BPF_MEM | BPF_W) || insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { type = BPF_READ; } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || insn->code == (BPF_STX | BPF_MEM | BPF_H) || insn->code == (BPF_STX | BPF_MEM | BPF_W) || insn->code == (BPF_STX | BPF_MEM | BPF_DW) || insn->code == (BPF_ST | BPF_MEM | BPF_B) || insn->code == (BPF_ST | BPF_MEM | BPF_H) || insn->code == (BPF_ST | BPF_MEM | BPF_W) || insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { type = BPF_WRITE; } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; continue; } else if (insn->code == (BPF_JMP | BPF_EXIT) && epilogue_cnt && i + delta < subprogs[1].start) { /* Generate epilogue for the main prog */ if (epilogue_idx) { /* jump back to the earlier generated epilogue */ insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); cnt = 1; } else { memcpy(insn_buf, epilogue_buf, epilogue_cnt * sizeof(*epilogue_buf)); cnt = epilogue_cnt; /* epilogue_idx cannot be 0. It must have at * least one ctx ptr saving insn before the * epilogue. */ epilogue_idx = i + delta; } goto patch_insn_buf; } else { continue; }
if (type == BPF_WRITE && env->insn_aux_data[i + delta].nospec_result) { /* nospec_result is only used to mitigate Spectre v4 and * to limit verification-time for Spectre v1. */ struct bpf_insn *patch = insn_buf;
*patch++ = *insn; *patch++ = BPF_ST_NOSPEC(); cnt = patch - insn_buf; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = new_prog; insn = new_prog->insnsi + i + delta; continue; }
switch ((int)env->insn_aux_data[i + delta].ptr_type) { case PTR_TO_CTX: if (!ops->convert_ctx_access) continue; convert_ctx_access = ops->convert_ctx_access; break; case PTR_TO_SOCKET: case PTR_TO_SOCK_COMMON: convert_ctx_access = bpf_sock_convert_ctx_access; break; case PTR_TO_TCP_SOCK: convert_ctx_access = bpf_tcp_sock_convert_ctx_access; break; case PTR_TO_XDP_SOCK: convert_ctx_access = bpf_xdp_sock_convert_ctx_access; break; case PTR_TO_BTF_ID: case PTR_TO_BTF_ID | PTR_UNTRUSTED: /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike * PTR_TO_BTF_ID, and an active referenced id, but the same cannot * be said once it is marked PTR_UNTRUSTED, hence we must handle * any faults for loads into such types. BPF_WRITE is disallowed * for this case. */ case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: case PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED: if (type == BPF_READ) { if (BPF_MODE(insn->code) == BPF_MEM) insn->code = BPF_LDX | BPF_PROBE_MEM | BPF_SIZE((insn)->code); else insn->code = BPF_LDX | BPF_PROBE_MEMSX | BPF_SIZE((insn)->code); env->prog->aux->num_exentries++; } continue; case PTR_TO_ARENA: if (BPF_MODE(insn->code) == BPF_MEMSX) { if (!bpf_jit_supports_insn(insn, true)) { verbose(env, "sign extending loads from arena are not supported yet\n"); return -EOPNOTSUPP; } insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code); } else { insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); } env->prog->aux->num_exentries++; continue; default: continue; }
ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; size = BPF_LDST_BYTES(insn); mode = BPF_MODE(insn->code);
/* If the read access is a narrower load of the field, * convert to a 4/8-byte load, to minimum program type specific * convert_ctx_access changes. If conversion is successful, * we will apply proper mask to the result. */ is_narrower_load = size < ctx_field_size; size_default = bpf_ctx_off_adjust_machine(ctx_field_size); off = insn->off; if (is_narrower_load) { u8 size_code;
if (type == BPF_WRITE) { verifier_bug(env, "narrow ctx access misconfigured"); return -EFAULT; }
size_code = BPF_H; if (ctx_field_size == 4) size_code = BPF_W; else if (ctx_field_size == 8) size_code = BPF_DW;
insn->off = off & ~(size_default - 1); insn->code = BPF_LDX | BPF_MEM | size_code; }
target_size = 0; cnt = convert_ctx_access(type, insn, insn_buf, env->prog, &target_size); if (cnt == 0 || cnt >= INSN_BUF_SIZE || (ctx_field_size && !target_size)) { verifier_bug(env, "error during ctx access conversion (%d)", cnt); return -EFAULT; }
if (is_narrower_load && size < target_size) { u8 shift = bpf_ctx_narrow_access_offset( off, size, size_default) * 8; if (shift && cnt + 1 >= INSN_BUF_SIZE) { verifier_bug(env, "narrow ctx load misconfigured"); return -EFAULT; } if (ctx_field_size <= 4) { if (shift) insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, insn->dst_reg, shift); insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, (1 << size * 8) - 1); } else { if (shift) insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, insn->dst_reg, shift); insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, (1ULL << size * 8) - 1); } } if (mode == BPF_MEMSX) insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, insn->dst_reg, insn->dst_reg, size * 8, 0);
patch_insn_buf: new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1;
/* keep walking new program and skip insns we just inserted */ env->prog = new_prog; insn = new_prog->insnsi + i + delta; }
return 0;}
static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env){ u32 *starts = NULL;
starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT); if (starts) { for (int i = 0; i < env->subprog_cnt; i++) starts[i] = env->subprog_info[i].start; } return starts;}
static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts){ for (int i = 0; i < env->subprog_cnt; i++) env->subprog_info[i].start = orig_starts[i]; /* restore the start of fake 'exit' subprog as well */ env->subprog_info[env->subprog_cnt].start = env->prog->len;}
struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env){ size_t size; void *new_aux;
size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len); new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT); if (new_aux) memcpy(new_aux, env->insn_aux_data, size); return new_aux;}
void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, struct bpf_insn_aux_data *orig_insn_aux){ /* the expanded elements are zero-filled, so no special handling is required */ vfree(env->insn_aux_data); env->insn_aux_data = orig_insn_aux;}
static int jit_subprogs(struct bpf_verifier_env *env){ struct bpf_prog *prog = env->prog, **func, *tmp; int i, j, subprog_start, subprog_end = 0, len, subprog; struct bpf_map *map_ptr; struct bpf_insn *insn; void *old_bpf_func; int err, num_exentries;
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) continue;
/* Upon error here we cannot fall back to interpreter but * need a hard reject of the program. Thus -EFAULT is * propagated in any case. */ subprog = bpf_find_subprog(env, i + insn->imm + 1); if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d", i + insn->imm + 1)) return -EFAULT; /* temporarily remember subprog id inside insn instead of * aux_data, since next loop will split up all insns into funcs */ insn->off = subprog; /* remember original imm in case JIT fails and fallback * to interpreter will be needed */ env->insn_aux_data[i].call_imm = insn->imm; /* point imm to __bpf_call_base+1 from JITs point of view */ insn->imm = 1; if (bpf_pseudo_func(insn)) {#if defined(MODULES_VADDR) u64 addr = MODULES_VADDR;#else u64 addr = VMALLOC_START;#endif /* jit (e.g. x86_64) may emit fewer instructions * if it learns a u32 imm is the same as a u64 imm. * Set close enough to possible prog address. */ insn[0].imm = (u32)addr; insn[1].imm = addr >> 32; } }
err = bpf_prog_alloc_jited_linfo(prog); if (err) goto out_undo_insn;
err = -ENOMEM; func = kzalloc_objs(prog, env->subprog_cnt); if (!func) goto out_undo_insn;
for (i = 0; i < env->subprog_cnt; i++) { subprog_start = subprog_end; subprog_end = env->subprog_info[i + 1].start;
len = subprog_end - subprog_start; /* bpf_prog_run() doesn't call subprogs directly, * hence main prog stats include the runtime of subprogs. * subprogs don't have IDs and not reachable via prog_get_next_id * func[i]->stats will never be accessed and stays NULL */ func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); if (!func[i]) goto out_free; memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], len * sizeof(struct bpf_insn)); func[i]->type = prog->type; func[i]->len = len; if (bpf_prog_calc_tag(func[i])) goto out_free; func[i]->is_func = 1; func[i]->sleepable = prog->sleepable; func[i]->blinded = prog->blinded; func[i]->aux->func_idx = i; /* Below members will be freed only at prog->aux */ func[i]->aux->btf = prog->aux->btf; func[i]->aux->subprog_start = subprog_start; func[i]->aux->func_info = prog->aux->func_info; func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; func[i]->aux->poke_tab = prog->aux->poke_tab; func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; func[i]->aux->main_prog_aux = prog->aux;
for (j = 0; j < prog->aux->size_poke_tab; j++) { struct bpf_jit_poke_descriptor *poke;
poke = &prog->aux->poke_tab[j]; if (poke->insn_idx < subprog_end && poke->insn_idx >= subprog_start) poke->aux = func[i]->aux; }
func[i]->aux->name[0] = 'F'; func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) func[i]->aux->jits_use_priv_stack = true;
func[i]->jit_requested = 1; func[i]->blinding_requested = prog->blinding_requested; func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; func[i]->aux->linfo = prog->aux->linfo; func[i]->aux->nr_linfo = prog->aux->nr_linfo; func[i]->aux->jited_linfo = prog->aux->jited_linfo; func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; func[i]->aux->arena = prog->aux->arena; func[i]->aux->used_maps = env->used_maps; func[i]->aux->used_map_cnt = env->used_map_cnt; num_exentries = 0; insn = func[i]->insnsi; for (j = 0; j < func[i]->len; j++, insn++) { if (BPF_CLASS(insn->code) == BPF_LDX && (BPF_MODE(insn->code) == BPF_PROBE_MEM || BPF_MODE(insn->code) == BPF_PROBE_MEM32 || BPF_MODE(insn->code) == BPF_PROBE_MEM32SX || BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) num_exentries++; if ((BPF_CLASS(insn->code) == BPF_STX || BPF_CLASS(insn->code) == BPF_ST) && BPF_MODE(insn->code) == BPF_PROBE_MEM32) num_exentries++; if (BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) num_exentries++; } func[i]->aux->num_exentries = num_exentries; func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; func[i]->aux->token = prog->aux->token; if (!i) func[i]->aux->exception_boundary = env->seen_exception; func[i] = bpf_int_jit_compile(env, func[i]); if (!func[i]->jited) { err = -ENOTSUPP; goto out_free; } cond_resched(); }
/* at this point all bpf functions were successfully JITed * now populate all bpf_calls with correct addresses and * run last pass of JIT */ for (i = 0; i < env->subprog_cnt; i++) { insn = func[i]->insnsi; for (j = 0; j < func[i]->len; j++, insn++) { if (bpf_pseudo_func(insn)) { subprog = insn->off; insn[0].imm = (u32)(long)func[subprog]->bpf_func; insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; continue; } if (!bpf_pseudo_call(insn)) continue; subprog = insn->off; insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); }
/* we use the aux data to keep a list of the start addresses * of the JITed images for each function in the program * * for some architectures, such as powerpc64, the imm field * might not be large enough to hold the offset of the start * address of the callee's JITed image from __bpf_call_base * * in such cases, we can lookup the start address of a callee * by using its subprog id, available from the off field of * the call instruction, as an index for this list */ func[i]->aux->func = func; func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; func[i]->aux->real_func_cnt = env->subprog_cnt; } for (i = 0; i < env->subprog_cnt; i++) { old_bpf_func = func[i]->bpf_func; tmp = bpf_int_jit_compile(env, func[i]); if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); err = -ENOTSUPP; goto out_free; } cond_resched(); }
/* * Cleanup func[i]->aux fields which aren't required * or can become invalid in future */ for (i = 0; i < env->subprog_cnt; i++) { func[i]->aux->used_maps = NULL; func[i]->aux->used_map_cnt = 0; }
/* finally lock prog and jit images for all functions and * populate kallsysm. Begin at the first subprogram, since * bpf_prog_load will add the kallsyms for the main program. */ for (i = 1; i < env->subprog_cnt; i++) { err = bpf_prog_lock_ro(func[i]); if (err) goto out_free; }
for (i = 1; i < env->subprog_cnt; i++) bpf_prog_kallsyms_add(func[i]);
/* Last step: make now unused interpreter insns from main * prog consistent for later dump requests, so they can * later look the same as if they were interpreted only. */ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { if (bpf_pseudo_func(insn)) { insn[0].imm = env->insn_aux_data[i].call_imm; insn[1].imm = insn->off; insn->off = 0; continue; } if (!bpf_pseudo_call(insn)) continue; insn->imm = env->insn_aux_data[i].call_imm; subprog = bpf_find_subprog(env, i + insn->imm + 1); insn->off = subprog; }
prog->jited = 1; prog->bpf_func = func[0]->bpf_func; prog->jited_len = func[0]->jited_len; prog->aux->extable = func[0]->aux->extable; prog->aux->num_exentries = func[0]->aux->num_exentries; prog->aux->func = func; prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; prog->aux->real_func_cnt = env->subprog_cnt; prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; prog->aux->exception_boundary = func[0]->aux->exception_boundary; prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust; bpf_prog_jit_attempt_done(prog); return 0;out_free: /* We failed JIT'ing, so at this point we need to unregister poke * descriptors from subprogs, so that kernel is not attempting to * patch it anymore as we're freeing the subprog JIT memory. */ for (i = 0; i < prog->aux->size_poke_tab; i++) { map_ptr = prog->aux->poke_tab[i].tail_call.map; map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); } /* At this point we're guaranteed that poke descriptors are not * live anymore. We can just unlink its descriptor table as it's * released with the main prog. */ for (i = 0; i < env->subprog_cnt; i++) { if (!func[i]) continue; func[i]->aux->poke_tab = NULL; bpf_jit_free(func[i]); } kfree(func);out_undo_insn: bpf_prog_jit_attempt_done(prog); return err;}
int bpf_jit_subprogs(struct bpf_verifier_env *env){ int err, i; bool blinded = false; struct bpf_insn *insn; struct bpf_prog *prog, *orig_prog; struct bpf_insn_aux_data *orig_insn_aux; u32 *orig_subprog_starts;
if (env->subprog_cnt <= 1) return 0;
prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { orig_insn_aux = bpf_dup_insn_aux_data(env); if (!orig_insn_aux) { err = -ENOMEM; goto out_cleanup; } orig_subprog_starts = bpf_dup_subprog_starts(env); if (!orig_subprog_starts) { vfree(orig_insn_aux); err = -ENOMEM; goto out_cleanup; } prog = bpf_jit_blind_constants(env, prog); if (IS_ERR(prog)) { err = -ENOMEM; prog = orig_prog; goto out_restore; } blinded = true; }
err = jit_subprogs(env); if (err) goto out_jit_err;
if (blinded) { bpf_jit_prog_release_other(prog, orig_prog); kvfree(orig_subprog_starts); vfree(orig_insn_aux); }
return 0;
out_jit_err: if (blinded) { bpf_jit_prog_release_other(orig_prog, prog); /* roll back to the clean original prog */ prog = env->prog = orig_prog; goto out_restore; } else { if (err != -EFAULT) { /* * We will fall back to interpreter mode when err is not -EFAULT, before * that, insn->off and insn->imm should be restored to their original * values since they were modified by jit_subprogs. */ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { if (!bpf_pseudo_call(insn)) continue; insn->off = 0; insn->imm = env->insn_aux_data[i].call_imm; } } goto out_cleanup; }
out_restore: bpf_restore_subprog_starts(env, orig_subprog_starts); bpf_restore_insn_aux_data(env, orig_insn_aux); kvfree(orig_subprog_starts);out_cleanup: /* cleanup main prog to be interpreted */ prog->jit_requested = 0; prog->blinding_requested = 0; return err;}
int bpf_fixup_call_args(struct bpf_verifier_env *env){#ifndef CONFIG_BPF_JIT_ALWAYS_ON struct bpf_prog *prog = env->prog; struct bpf_insn *insn = prog->insnsi; bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); int depth;#endif int i, err = 0;
for (i = 0; i < env->subprog_cnt; i++) { struct bpf_subprog_info *subprog = &env->subprog_info[i]; u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog);
if (subprog->max_out_stack_arg_cnt > outgoing) { verbose(env, "func#%d writes %u stack arg slots, but calls only require %u\n", i, subprog->max_out_stack_arg_cnt, outgoing); return -EINVAL; } }
if (env->prog->jit_requested && !bpf_prog_is_offloaded(env->prog->aux)) { err = bpf_jit_subprogs(env); if (err == 0) return 0; if (err == -EFAULT) return err; }#ifndef CONFIG_BPF_JIT_ALWAYS_ON if (has_kfunc_call) { verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); return -EINVAL; } for (i = 0; i < env->subprog_cnt; i++) { if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { verbose(env, "stack args are not supported in non-JITed programs\n"); return -EINVAL; } } if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { /* When JIT fails the progs with bpf2bpf calls and tail_calls * have to be rejected, since interpreter doesn't support them yet. */ verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); return -EINVAL; } for (i = 0; i < prog->len; i++, insn++) { if (bpf_pseudo_func(insn)) { /* When JIT fails the progs with callback calls * have to be rejected, since interpreter doesn't support them yet. */ verbose(env, "callbacks are not allowed in non-JITed programs\n"); return -EINVAL; }
if (!bpf_pseudo_call(insn)) continue; depth = get_callee_stack_depth(env, insn, i); if (depth < 0) return depth; err = bpf_patch_call_args(insn, depth); if (err) { verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", depth); return err; } } err = 0;#endif return err;}
/* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len){ struct bpf_subprog_info *info = env->subprog_info; int cnt = env->subprog_cnt; struct bpf_prog *prog;
/* We only reserve one slot for hidden subprogs in subprog_info. */ if (env->hidden_subprog_cnt) { verifier_bug(env, "only one hidden subprog supported"); return -EFAULT; } /* We're not patching any existing instruction, just appending the new * ones for the hidden subprog. Hence all of the adjustment operations * in bpf_patch_insn_data are no-ops. */ prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); if (!prog) return -ENOMEM; env->prog = prog; info[cnt + 1].start = info[cnt].start; info[cnt].start = prog->len - len + 1; env->subprog_cnt++; env->hidden_subprog_cnt++; return 0;}
/* Do various post-verification rewrites in a single program pass. * These rewrites simplify JIT and interpreter implementations. */int bpf_do_misc_fixups(struct bpf_verifier_env *env){ struct bpf_prog *prog = env->prog; enum bpf_attach_type eatype = prog->expected_attach_type; enum bpf_prog_type prog_type = resolve_prog_type(prog); struct bpf_insn *insn = prog->insnsi; const struct bpf_func_proto *fn; const int insn_cnt = prog->len; const struct bpf_map_ops *ops; struct bpf_insn_aux_data *aux; struct bpf_insn *insn_buf = env->insn_buf; struct bpf_prog *new_prog; struct bpf_map *map_ptr; int i, ret, cnt, delta = 0, cur_subprog = 0; struct bpf_subprog_info *subprogs = env->subprog_info; u16 stack_depth = subprogs[cur_subprog].stack_depth; u16 stack_depth_extra = 0;
if (env->seen_exception && !env->exception_callback_subprog) { struct bpf_insn *patch = insn_buf;
*patch++ = env->prog->insnsi[insn_cnt - 1]; *patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); *patch++ = BPF_EXIT_INSN(); ret = add_hidden_subprog(env, insn_buf, patch - insn_buf); if (ret < 0) return ret; prog = env->prog; insn = prog->insnsi;
env->exception_callback_subprog = env->subprog_cnt - 1; /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog); }
for (i = 0; i < insn_cnt;) { if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) { if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) || (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { /* convert to 32-bit mov that clears upper 32-bit */ insn->code = BPF_ALU | BPF_MOV | BPF_X; /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ insn->off = 0; insn->imm = 0; } /* cast from as(0) to as(1) should be handled by JIT */ goto next_insn; }
if (env->insn_aux_data[i + delta].needs_zext) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code);
/* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || insn->code == (BPF_ALU | BPF_MOD | BPF_K) || insn->code == (BPF_ALU | BPF_DIV | BPF_K)) && insn->off == 1 && insn->imm == -1) { bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; bool isdiv = BPF_OP(insn->code) == BPF_DIV; struct bpf_insn *patch = insn_buf;
if (isdiv) *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | BPF_NEG | BPF_K, insn->dst_reg, 0, 0, 0); else *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0);
cnt = patch - insn_buf;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Make divide-by-zero and divide-by-minus-one exceptions impossible. */ if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || insn->code == (BPF_ALU | BPF_MOD | BPF_X) || insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; bool isdiv = BPF_OP(insn->code) == BPF_DIV; bool is_sdiv = isdiv && insn->off == 1; bool is_smod = !isdiv && insn->off == 1; struct bpf_insn *patch = insn_buf;
if (is_sdiv) { /* [R,W]x sdiv 0 -> 0 * LLONG_MIN sdiv -1 -> LLONG_MIN * INT_MIN sdiv -1 -> INT_MIN */ *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | BPF_ADD | BPF_K, BPF_REG_AX, 0, 0, 1); *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | BPF_JGT | BPF_K, BPF_REG_AX, 0, 4, 1); *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | BPF_JEQ | BPF_K, BPF_REG_AX, 0, 1, 0); *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | BPF_MOV | BPF_K, insn->dst_reg, 0, 0, 0); /* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */ *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | BPF_NEG | BPF_K, insn->dst_reg, 0, 0, 0); *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = *insn; cnt = patch - insn_buf; } else if (is_smod) { /* [R,W]x mod 0 -> [R,W]x */ /* [R,W]x mod -1 -> 0 */ *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); *patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) | BPF_ADD | BPF_K, BPF_REG_AX, 0, 0, 1); *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | BPF_JGT | BPF_K, BPF_REG_AX, 0, 3, 1); *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | BPF_JEQ | BPF_K, BPF_REG_AX, 0, 3 + (is64 ? 0 : 1), 1); *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = *insn;
if (!is64) { *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); } cnt = patch - insn_buf; } else if (isdiv) { /* [R,W]x div 0 -> 0 */ *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | BPF_JNE | BPF_K, insn->src_reg, 0, 2, 0); *patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg); *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = *insn; cnt = patch - insn_buf; } else { /* [R,W]x mod 0 -> [R,W]x */ *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | BPF_JEQ | BPF_K, insn->src_reg, 0, 1 + (is64 ? 0 : 1), 0); *patch++ = *insn;
if (!is64) { *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg); } cnt = patch - insn_buf; }
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Make it impossible to de-reference a userspace address */ if (BPF_CLASS(insn->code) == BPF_LDX && (BPF_MODE(insn->code) == BPF_PROBE_MEM || BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { struct bpf_insn *patch = insn_buf; u64 uaddress_limit = bpf_arch_uaddress_limit();
if (!uaddress_limit) goto next_insn;
*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); if (insn->off) *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); *patch++ = *insn; *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);
cnt = patch - insn_buf; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ if (BPF_CLASS(insn->code) == BPF_LD && (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND)) { cnt = env->ops->gen_ld_abs(insn, insn_buf); if (cnt == 0 || cnt >= INSN_BUF_SIZE) { verifier_bug(env, "%d insns generated for ld_abs", cnt); return -EFAULT; }
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Rewrite pointer arithmetic to mitigate speculation attacks. */ if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; struct bpf_insn *patch = insn_buf; bool issrc, isneg, isimm; u32 off_reg;
aux = &env->insn_aux_data[i + delta]; if (!aux->alu_state || aux->alu_state == BPF_ALU_NON_POINTER) goto next_insn;
isneg = aux->alu_state & BPF_ALU_NEG_VALUE; issrc = (aux->alu_state & BPF_ALU_SANITIZE) == BPF_ALU_SANITIZE_SRC; isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
off_reg = issrc ? insn->src_reg : insn->dst_reg; if (isimm) { *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); } else { if (isneg) *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); } if (!issrc) *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); insn->src_reg = BPF_REG_AX; if (isneg) insn->code = insn->code == code_add ? code_sub : code_add; *patch++ = *insn; if (issrc && isneg && !isimm) *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); cnt = patch - insn_buf;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { int stack_off_cnt = -stack_depth - 16;
/* * Two 8 byte slots, depth-16 stores the count, and * depth-8 stores the start timestamp of the loop. * * The starting value of count is BPF_MAX_TIMED_LOOPS * (0xffff). Every iteration loads it and subs it by 1, * until the value becomes 0 in AX (thus, 1 in stack), * after which we call arch_bpf_timed_may_goto, which * either sets AX to 0xffff to keep looping, or to 0 * upon timeout. AX is then stored into the stack. In * the next iteration, we either see 0 and break out, or * continue iterating until the next time value is 0 * after subtraction, rinse and repeat. */ stack_depth_extra = 16; insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); if (insn->off >= 0) insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); else insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); /* * AX is used as an argument to pass in stack_off_cnt * (to add to r10/fp), and also as the return value of * the call to arch_bpf_timed_may_goto. */ insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); cnt = 7;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; } else if (bpf_is_may_goto_insn(insn)) { int stack_off = -stack_depth - 8;
stack_depth_extra = 8; insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); if (insn->off >= 0) insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); else insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); cnt = 4;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
if (insn->code != (BPF_JMP | BPF_CALL)) goto next_insn; if (insn->src_reg == BPF_PSEUDO_CALL) goto next_insn; if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); if (ret) return ret; if (cnt == 0) goto next_insn;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Skip inlining the helper call if the JIT does it. */ if (bpf_jit_inlines_helper_call(insn->imm)) goto next_insn;
if (insn->imm == BPF_FUNC_get_route_realm) prog->dst_needed = 1; if (insn->imm == BPF_FUNC_get_prandom_u32) bpf_user_rnd_init_once(); if (insn->imm == BPF_FUNC_override_return) prog->kprobe_override = 1; if (insn->imm == BPF_FUNC_tail_call) { /* If we tail call into other programs, we * cannot make any assumptions since they can * be replaced dynamically during runtime in * the program array. */ prog->cb_access = 1; if (!bpf_allow_tail_call_in_subprogs(env)) prog->aux->stack_depth = MAX_BPF_STACK; prog->aux->max_pkt_offset = MAX_PACKET_OFF;
/* mark bpf_tail_call as different opcode to avoid * conditional branch in the interpreter for every normal * call and to prevent accidental JITing by JIT compiler * that doesn't support bpf_tail_call yet */ insn->imm = 0; insn->code = BPF_JMP | BPF_TAIL_CALL;
aux = &env->insn_aux_data[i + delta]; if (env->bpf_capable && !prog->blinding_requested && prog->jit_requested && !bpf_map_key_poisoned(aux) && !bpf_map_ptr_poisoned(aux) && !bpf_map_ptr_unpriv(aux)) { struct bpf_jit_poke_descriptor desc = { .reason = BPF_POKE_REASON_TAIL_CALL, .tail_call.map = aux->map_ptr_state.map_ptr, .tail_call.key = bpf_map_key_immediate(aux), .insn_idx = i + delta, };
ret = bpf_jit_add_poke_descriptor(prog, &desc); if (ret < 0) { verbose(env, "adding tail call poke descriptor failed\n"); return ret; }
insn->imm = ret + 1; goto next_insn; }
if (!bpf_map_ptr_unpriv(aux)) goto next_insn;
/* instead of changing every JIT dealing with tail_call * emit two extra insns: * if (index >= max_entries) goto out; * index &= array->index_mask; * to avoid out-of-bounds cpu speculation */ if (bpf_map_ptr_poisoned(aux)) { verbose(env, "tail_call abusing map_ptr\n"); return -EINVAL; }
map_ptr = aux->map_ptr_state.map_ptr; insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, map_ptr->max_entries, 2); insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, container_of(map_ptr, struct bpf_array, map)->index_mask); insn_buf[2] = *insn; cnt = 3; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
if (insn->imm == BPF_FUNC_timer_set_callback) { /* The verifier will process callback_fn as many times as necessary * with different maps and the register states prepared by * set_timer_callback_state will be accurate. * * The following use case is valid: * map1 is shared by prog1, prog2, prog3. * prog1 calls bpf_timer_init for some map1 elements * prog2 calls bpf_timer_set_callback for some map1 elements. * Those that were not bpf_timer_init-ed will return -EINVAL. * prog3 calls bpf_timer_start for some map1 elements. * Those that were not both bpf_timer_init-ed and * bpf_timer_set_callback-ed will return -EINVAL. */ struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), };
insn_buf[0] = ld_addrs[0]; insn_buf[1] = ld_addrs[1]; insn_buf[2] = *insn; cnt = 3;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto patch_call_imm; }
/* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, * bpf_mem_alloc() returns a ptr to the percpu data ptr. */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); insn_buf[1] = *insn; cnt = 2;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto patch_call_imm; }
/* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup * and other inlining handlers are currently limited to 64 bit * only. */ if (prog->jit_requested && BITS_PER_LONG == 64 && (insn->imm == BPF_FUNC_map_lookup_elem || insn->imm == BPF_FUNC_map_update_elem || insn->imm == BPF_FUNC_map_delete_elem || insn->imm == BPF_FUNC_map_push_elem || insn->imm == BPF_FUNC_map_pop_elem || insn->imm == BPF_FUNC_map_peek_elem || insn->imm == BPF_FUNC_redirect_map || insn->imm == BPF_FUNC_for_each_map_elem || insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { aux = &env->insn_aux_data[i + delta]; if (bpf_map_ptr_poisoned(aux)) goto patch_call_imm;
map_ptr = aux->map_ptr_state.map_ptr; ops = map_ptr->ops; if (insn->imm == BPF_FUNC_map_lookup_elem && ops->map_gen_lookup) { cnt = ops->map_gen_lookup(map_ptr, insn_buf); if (cnt == -EOPNOTSUPP) goto patch_map_ops_generic; if (cnt <= 0 || cnt >= INSN_BUF_SIZE) { verifier_bug(env, "%d insns generated for map lookup", cnt); return -EFAULT; }
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); BUILD_BUG_ON(!__same_type(ops->map_delete_elem, (long (*)(struct bpf_map *map, void *key))NULL)); BUILD_BUG_ON(!__same_type(ops->map_update_elem, (long (*)(struct bpf_map *map, void *key, void *value, u64 flags))NULL)); BUILD_BUG_ON(!__same_type(ops->map_push_elem, (long (*)(struct bpf_map *map, void *value, u64 flags))NULL)); BUILD_BUG_ON(!__same_type(ops->map_pop_elem, (long (*)(struct bpf_map *map, void *value))NULL)); BUILD_BUG_ON(!__same_type(ops->map_peek_elem, (long (*)(struct bpf_map *map, void *value))NULL)); BUILD_BUG_ON(!__same_type(ops->map_redirect, (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, (long (*)(struct bpf_map *map, bpf_callback_t callback_fn, void *callback_ctx, u64 flags))NULL)); BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL));
patch_map_ops_generic: switch (insn->imm) { case BPF_FUNC_map_lookup_elem: insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); goto next_insn; case BPF_FUNC_map_update_elem: insn->imm = BPF_CALL_IMM(ops->map_update_elem); goto next_insn; case BPF_FUNC_map_delete_elem: insn->imm = BPF_CALL_IMM(ops->map_delete_elem); goto next_insn; case BPF_FUNC_map_push_elem: insn->imm = BPF_CALL_IMM(ops->map_push_elem); goto next_insn; case BPF_FUNC_map_pop_elem: insn->imm = BPF_CALL_IMM(ops->map_pop_elem); goto next_insn; case BPF_FUNC_map_peek_elem: insn->imm = BPF_CALL_IMM(ops->map_peek_elem); goto next_insn; case BPF_FUNC_redirect_map: insn->imm = BPF_CALL_IMM(ops->map_redirect); goto next_insn; case BPF_FUNC_for_each_map_elem: insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); goto next_insn; case BPF_FUNC_map_lookup_percpu_elem: insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); goto next_insn; }
goto patch_call_imm; }
/* Implement bpf_jiffies64 inline. */ if (prog->jit_requested && BITS_PER_LONG == 64 && insn->imm == BPF_FUNC_jiffies64) { struct bpf_insn ld_jiffies_addr[2] = { BPF_LD_IMM64(BPF_REG_0, (unsigned long)&jiffies), };
insn_buf[0] = ld_jiffies_addr[0]; insn_buf[1] = ld_jiffies_addr[1]; insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); cnt = 3;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) /* Implement bpf_get_smp_processor_id() inline. */ if (insn->imm == BPF_FUNC_get_smp_processor_id && bpf_verifier_inlines_helper_call(env, insn->imm)) { /* BPF_FUNC_get_smp_processor_id inlining is an * optimization, so if cpu_number is ever * changed in some incompatible and hard to support * way, it's fine to back out this inlining logic */#ifdef CONFIG_SMP insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); cnt = 3;#else insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); cnt = 1;#endif new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && bpf_verifier_inlines_helper_call(env, insn->imm)) { insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); cnt = 3;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }#endif /* Implement bpf_get_func_arg inline. */ if (prog_type == BPF_PROG_TYPE_TRACING && insn->imm == BPF_FUNC_get_func_arg) { if (eatype == BPF_TRACE_RAW_TP) { int nr_args = btf_type_vlen(prog->aux->attach_func_proto);
/* skip 'void *__data' in btf_trace_##name() and save to reg0 */ insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); cnt = 1; } else { /* Load nr_args from ctx - 8 */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); cnt = 2; } insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 0); insn_buf[cnt++] = BPF_JMP_A(1); insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Implement bpf_get_func_ret inline. */ if (prog_type == BPF_PROG_TYPE_TRACING && insn->imm == BPF_FUNC_get_func_ret) { if (eatype == BPF_TRACE_FEXIT || eatype == BPF_TRACE_FSESSION || eatype == BPF_TRACE_FEXIT_MULTI || eatype == BPF_TRACE_FSESSION_MULTI || eatype == BPF_MODIFY_RETURN) { /* Load nr_args from ctx - 8 */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); cnt = 7; } else { insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); cnt = 1; }
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Implement get_func_arg_cnt inline. */ if (prog_type == BPF_PROG_TYPE_TRACING && insn->imm == BPF_FUNC_get_func_arg_cnt) { if (eatype == BPF_TRACE_RAW_TP) { int nr_args = btf_type_vlen(prog->aux->attach_func_proto);
/* skip 'void *__data' in btf_trace_##name() and save to reg0 */ insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, nr_args - 1); cnt = 1; } else { /* Load nr_args from ctx - 8 */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); insn_buf[1] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); cnt = 2; }
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Implement bpf_get_func_ip inline. */ if (prog_type == BPF_PROG_TYPE_TRACING && insn->imm == BPF_FUNC_get_func_ip) { /* Load IP address from ctx - 16 */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); if (!new_prog) return -ENOMEM;
env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Implement bpf_get_branch_snapshot inline. */ if (IS_ENABLED(CONFIG_PERF_EVENTS) && prog->jit_requested && BITS_PER_LONG == 64 && insn->imm == BPF_FUNC_get_branch_snapshot) { /* We are dealing with the following func protos: * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags); * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt); */ const u32 br_entry_size = sizeof(struct perf_branch_entry);
/* struct perf_branch_entry is part of UAPI and is * used as an array element, so extremely unlikely to * ever grow or shrink */ BUILD_BUG_ON(br_entry_size != 24);
/* if (unlikely(flags)) return -EINVAL */ insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7);
/* Transform size (bytes) into number of entries (cnt = size / 24). * But to avoid expensive division instruction, we implement * divide-by-3 through multiplication, followed by further * division by 8 through 3-bit right shift. * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr., * p. 227, chapter "Unsigned Division by 3" for details and proofs. * * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab. */ insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36);
/* call perf_snapshot_branch_stack implementation */ insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); /* if (entry_cnt == 0) return -ENOENT */ insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); /* return entry_cnt * sizeof(struct perf_branch_entry) */ insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); insn_buf[7] = BPF_JMP_A(3); /* return -EINVAL; */ insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); insn_buf[9] = BPF_JMP_A(1); /* return -ENOENT; */ insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); cnt = 11;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }
/* Implement bpf_kptr_xchg inline */ if (prog->jit_requested && BITS_PER_LONG == 64 && insn->imm == BPF_FUNC_kptr_xchg && bpf_jit_supports_ptr_xchg()) { insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); cnt = 2;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; goto next_insn; }patch_call_imm: fn = env->ops->get_func_proto(insn->imm, env->prog); /* all functions that have prototype and verifier allowed * programs to call them, must be real in-kernel functions */ if (!fn->func) { verifier_bug(env, "not inlined functions %s#%d is missing func", func_id_name(insn->imm), insn->imm); return -EFAULT; } insn->imm = fn->func - __bpf_call_base;next_insn: if (subprogs[cur_subprog + 1].start == i + delta + 1) { subprogs[cur_subprog].stack_depth += stack_depth_extra; subprogs[cur_subprog].stack_extra = stack_depth_extra;
stack_depth = subprogs[cur_subprog].stack_depth; if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) { verbose(env, "stack size %d(extra %d) is too large\n", stack_depth, stack_depth_extra); return -EINVAL; } cur_subprog++; stack_depth = subprogs[cur_subprog].stack_depth; stack_depth_extra = 0; } i++; insn++; }
env->prog->aux->stack_depth = subprogs[0].stack_depth; for (i = 0; i < env->subprog_cnt; i++) { int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; int subprog_start = subprogs[i].start; int stack_slots = subprogs[i].stack_extra / 8; int slots = delta, cnt = 0;
if (!stack_slots) continue; /* We need two slots in case timed may_goto is supported. */ if (stack_slots > slots) { verifier_bug(env, "stack_slots supports may_goto only"); return -EFAULT; }
stack_depth = subprogs[i].stack_depth; if (bpf_jit_supports_timed_may_goto()) { insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, BPF_MAX_TIMED_LOOPS); insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); } else { /* Add ST insn to subprog prologue to init extra stack */ insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, BPF_MAX_LOOPS); } /* Copy first actual insn to preserve it */ insn_buf[cnt++] = env->prog->insnsi[subprog_start];
new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); if (!new_prog) return -ENOMEM; env->prog = prog = new_prog; /* * If may_goto is a first insn of a prog there could be a jmp * insn that points to it, hence adjust all such jmps to point * to insn after BPF_ST that inits may_goto count. * Adjustment will succeed because bpf_patch_insn_data() didn't fail. */ WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); }
/* Since poke tab is now finalized, publish aux to tracker. */ for (i = 0; i < prog->aux->size_poke_tab; i++) { map_ptr = prog->aux->poke_tab[i].tail_call.map; if (!map_ptr->ops->map_poke_track || !map_ptr->ops->map_poke_untrack || !map_ptr->ops->map_poke_run) { verifier_bug(env, "poke tab is misconfigured"); return -EFAULT; }
ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); if (ret < 0) { verbose(env, "tracking tail call prog failed\n"); return ret; } }
ret = sort_kfunc_descs_by_imm_off(env); if (ret) return ret;
return 0;}
static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, int position, s32 stack_base, u32 callback_subprogno, u32 *total_cnt){ s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; int reg_loop_max = BPF_REG_6; int reg_loop_cnt = BPF_REG_7; int reg_loop_ctx = BPF_REG_8;
struct bpf_insn *insn_buf = env->insn_buf; struct bpf_prog *new_prog; u32 callback_start; u32 call_insn_offset; s32 callback_offset; u32 cnt = 0;
/* This represents an inlined version of bpf_iter.c:bpf_loop, * be careful to modify this code in sync. */
/* Return error and jump to the end of the patch if * expected number of iterations is too big. */ insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2); insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG); insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16); /* spill R6, R7, R8 to use these as loop vars */ insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset); insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset); insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset); /* initialize loop vars */ insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1); insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0); insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3); /* loop header, * if reg_loop_cnt >= reg_loop_max skip the loop body */ insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); /* callback call, * correct callback offset would be set after patching */ insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); insn_buf[cnt++] = BPF_CALL_REL(0); /* increment loop counter */ insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1); /* jump to loop header if callback returned 0 */ insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6); /* return value of bpf_loop, * set R0 to the number of iterations */ insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt); /* restore original values of R6, R7, R8 */ insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset); insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset);
*total_cnt = cnt; new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); if (!new_prog) return new_prog;
/* callback start is known only after patching */ callback_start = env->subprog_info[callback_subprogno].start; /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ call_insn_offset = position + 12; callback_offset = callback_start - call_insn_offset - 1; new_prog->insnsi[call_insn_offset].imm = callback_offset;
return new_prog;}
static bool is_bpf_loop_call(struct bpf_insn *insn){ return insn->code == (BPF_JMP | BPF_CALL) && insn->src_reg == 0 && insn->imm == BPF_FUNC_loop;}
/* For all sub-programs in the program (including main) check * insn_aux_data to see if there are bpf_loop calls that require * inlining. If such calls are found the calls are replaced with a * sequence of instructions produced by `inline_bpf_loop` function and * subprog stack_depth is increased by the size of 3 registers. * This stack space is used to spill values of the R6, R7, R8. These * registers are used to store the loop bound, counter and context * variables. */int bpf_optimize_bpf_loop(struct bpf_verifier_env *env){ struct bpf_subprog_info *subprogs = env->subprog_info; int i, cur_subprog = 0, cnt, delta = 0; struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; u16 stack_depth = subprogs[cur_subprog].stack_depth; u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; u16 stack_depth_extra = 0;
for (i = 0; i < insn_cnt; i++, insn++) { struct bpf_loop_inline_state *inline_state = &env->insn_aux_data[i + delta].loop_inline_state;
if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { struct bpf_prog *new_prog;
stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; new_prog = inline_bpf_loop(env, i + delta, -(stack_depth + stack_depth_extra), inline_state->callback_subprogno, &cnt); if (!new_prog) return -ENOMEM;
delta += cnt - 1; env->prog = new_prog; insn = new_prog->insnsi + i + delta; }
if (subprogs[cur_subprog + 1].start == i + delta + 1) { subprogs[cur_subprog].stack_depth += stack_depth_extra; cur_subprog++; stack_depth = subprogs[cur_subprog].stack_depth; stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; stack_depth_extra = 0; } }
env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
return 0;}
/* Remove unnecessary spill/fill pairs, members of fastcall pattern, * adjust subprograms stack depth when possible. */int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env){ struct bpf_subprog_info *subprog = env->subprog_info; struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; u32 spills_num; bool modified = false; int i, j;
for (i = 0; i < insn_cnt; i++, insn++) { if (aux[i].fastcall_spills_num > 0) { spills_num = aux[i].fastcall_spills_num; /* NOPs would be removed by opt_remove_nops() */ for (j = 1; j <= spills_num; ++j) { *(insn - j) = NOP; *(insn + j) = NOP; } modified = true; } if ((subprog + 1)->start == i + 1) { if (modified && !subprog->keep_fastcall_stack) subprog->stack_depth = -subprog->fastcall_stack_off; subprog++; modified = false; } }
return 0;}