bpf: Support new signed div/mod instructions.

Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git

kernel os linux

Add interpreter/jit support for new signed div/mod insns.
The new signed div/mod instructions are encoded with
unsigned div/mod instructions plus insn->off == 1.
Also add basic verifier support to ensure new insns get
accepted.

Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20230728011219.3714605-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

authored by

Yonghong Song and committed by

Alexei Starovoitov 2 years ago ec0e2da9 0845c3db

+117 -26

3 changed files

expand all

arch

x86

net

bpf_jit_comp.c

kernel

bpf

core.c

verifier.c

+19 -8

arch/x86/net/bpf_jit_comp.c

··· 1194 1194 /* mov rax, dst_reg */ 1195 1195 emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg); 1196 1196 1197 - /* 1198 - * xor edx, edx 1199 - * equivalent to 'xor rdx, rdx', but one byte less 1200 - */ 1201 - EMIT2(0x31, 0xd2); 1197 + if (insn->off == 0) { 1198 + /* 1199 + * xor edx, edx 1200 + * equivalent to 'xor rdx, rdx', but one byte less 1201 + */ 1202 + EMIT2(0x31, 0xd2); 1202 1203 1203 - /* div src_reg */ 1204 - maybe_emit_1mod(&prog, src_reg, is64); 1205 - EMIT2(0xF7, add_1reg(0xF0, src_reg)); 1204 + /* div src_reg */ 1205 + maybe_emit_1mod(&prog, src_reg, is64); 1206 + EMIT2(0xF7, add_1reg(0xF0, src_reg)); 1207 + } else { 1208 + if (BPF_CLASS(insn->code) == BPF_ALU) 1209 + EMIT1(0x99); /* cdq */ 1210 + else 1211 + EMIT2(0x48, 0x99); /* cqo */ 1212 + 1213 + /* idiv src_reg */ 1214 + maybe_emit_1mod(&prog, src_reg, is64); 1215 + EMIT2(0xF7, add_1reg(0xF8, src_reg)); 1216 + } 1206 1217 1207 1218 if (BPF_OP(insn->code) == BPF_MOD && 1208 1219 dst_reg != BPF_REG_3)

+94 -16

kernel/bpf/core.c

··· 1792 1792 (*(s64 *) &DST) >>= IMM; 1793 1793 CONT; 1794 1794 ALU64_MOD_X: 1795 - div64_u64_rem(DST, SRC, &AX); 1796 - DST = AX; 1795 + switch (OFF) { 1796 + case 0: 1797 + div64_u64_rem(DST, SRC, &AX); 1798 + DST = AX; 1799 + break; 1800 + case 1: 1801 + AX = div64_s64(DST, SRC); 1802 + DST = DST - AX * SRC; 1803 + break; 1804 + } 1797 1805 CONT; 1798 1806 ALU_MOD_X: 1799 - AX = (u32) DST; 1800 - DST = do_div(AX, (u32) SRC); 1807 + switch (OFF) { 1808 + case 0: 1809 + AX = (u32) DST; 1810 + DST = do_div(AX, (u32) SRC); 1811 + break; 1812 + case 1: 1813 + AX = abs((s32)DST); 1814 + AX = do_div(AX, abs((s32)SRC)); 1815 + if ((s32)DST < 0) 1816 + DST = (u32)-AX; 1817 + else 1818 + DST = (u32)AX; 1819 + break; 1820 + } 1801 1821 CONT; 1802 1822 ALU64_MOD_K: 1803 - div64_u64_rem(DST, IMM, &AX); 1804 - DST = AX; 1823 + switch (OFF) { 1824 + case 0: 1825 + div64_u64_rem(DST, IMM, &AX); 1826 + DST = AX; 1827 + break; 1828 + case 1: 1829 + AX = div64_s64(DST, IMM); 1830 + DST = DST - AX * IMM; 1831 + break; 1832 + } 1805 1833 CONT; 1806 1834 ALU_MOD_K: 1807 - AX = (u32) DST; 1808 - DST = do_div(AX, (u32) IMM); 1835 + switch (OFF) { 1836 + case 0: 1837 + AX = (u32) DST; 1838 + DST = do_div(AX, (u32) IMM); 1839 + break; 1840 + case 1: 1841 + AX = abs((s32)DST); 1842 + AX = do_div(AX, abs((s32)IMM)); 1843 + if ((s32)DST < 0) 1844 + DST = (u32)-AX; 1845 + else 1846 + DST = (u32)AX; 1847 + break; 1848 + } 1809 1849 CONT; 1810 1850 ALU64_DIV_X: 1811 - DST = div64_u64(DST, SRC); 1851 + switch (OFF) { 1852 + case 0: 1853 + DST = div64_u64(DST, SRC); 1854 + break; 1855 + case 1: 1856 + DST = div64_s64(DST, SRC); 1857 + break; 1858 + } 1812 1859 CONT; 1813 1860 ALU_DIV_X: 1814 - AX = (u32) DST; 1815 - do_div(AX, (u32) SRC); 1816 - DST = (u32) AX; 1861 + switch (OFF) { 1862 + case 0: 1863 + AX = (u32) DST; 1864 + do_div(AX, (u32) SRC); 1865 + DST = (u32) AX; 1866 + break; 1867 + case 1: 1868 + AX = abs((s32)DST); 1869 + do_div(AX, abs((s32)SRC)); 1870 + if ((s32)DST < 0 == (s32)SRC < 0) 1871 + DST = (u32)AX; 1872 + else 1873 + DST = (u32)-AX; 1874 + break; 1875 + } 1817 1876 CONT; 1818 1877 ALU64_DIV_K: 1819 - DST = div64_u64(DST, IMM); 1878 + switch (OFF) { 1879 + case 0: 1880 + DST = div64_u64(DST, IMM); 1881 + break; 1882 + case 1: 1883 + DST = div64_s64(DST, IMM); 1884 + break; 1885 + } 1820 1886 CONT; 1821 1887 ALU_DIV_K: 1822 - AX = (u32) DST; 1823 - do_div(AX, (u32) IMM); 1824 - DST = (u32) AX; 1888 + switch (OFF) { 1889 + case 0: 1890 + AX = (u32) DST; 1891 + do_div(AX, (u32) IMM); 1892 + DST = (u32) AX; 1893 + break; 1894 + case 1: 1895 + AX = abs((s32)DST); 1896 + do_div(AX, abs((s32)IMM)); 1897 + if ((s32)DST < 0 == (s32)IMM < 0) 1898 + DST = (u32)AX; 1899 + else 1900 + DST = (u32)-AX; 1901 + break; 1902 + } 1825 1903 CONT; 1826 1904 ALU_END_TO_BE: 1827 1905 switch (IMM) {

+4 -2

kernel/bpf/verifier.c

··· 13237 13237 } else { /* all other ALU ops: and, sub, xor, add, ... */ 13238 13238 13239 13239 if (BPF_SRC(insn->code) == BPF_X) { 13240 - if (insn->imm != 0 || insn->off != 0) { 13240 + if (insn->imm != 0 || insn->off > 1 || 13241 + (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { 13241 13242 verbose(env, "BPF_ALU uses reserved fields\n"); 13242 13243 return -EINVAL; 13243 13244 } ··· 13247 13246 if (err) 13248 13247 return err; 13249 13248 } else { 13250 - if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 13249 + if (insn->src_reg != BPF_REG_0 || insn->off > 1 || 13250 + (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { 13251 13251 verbose(env, "BPF_ALU uses reserved fields\n"); 13252 13252 return -EINVAL; 13253 13253 }