Something went wrong. Try again.
llama.cpp fork for Ternary Bonsai 2 27B: MTP speculation, suffix cache reuse (SCR), q8 KV, two-slot fairness, fused DeltaNet kernels
Something went wrong. Try again.
21 kB · 471 lines
C++
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472#include "peg-parser.h"#include "tests.h"
void test_basic(testing & t) { t.test("chars", [](testing & t) { // Test common escape sequences - newline t.test("escape_sequence_newline", [](testing &t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("\n"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escape_sequence_newline", true, result.success()); });
// Test common escape sequences - tab t.test("escape_sequence_tab", [](testing &t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("\t"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escape_sequence_tab", true, result.success()); });
// Test common escape sequences - backslash t.test("escape_sequence_backslash", [](testing &t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("\\"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escape_sequence_backslash", true, result.success()); });
// Test common escape sequences - space (should ()) t.test("escape_sequence_space_fail", [](testing &t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context(" "); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escape_sequence_space_fail", true, result.fail()); });
// Test escaped dash - 'a' should succeed t.test("escaped_dash_a", [](testing &t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("a"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escaped_dash_a", true, result.success()); });
// Test escaped dash - '-' should succeed (literal dash) t.test("escaped_dash_literal", [](testing &t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("-"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escaped_dash_literal", true, result.success()); });
// Test escaped dash - 'z' should succeed t.test("escaped_dash_z", [](testing &t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("z"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escaped_dash_z", true, result.success()); });
// Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) t.test("escaped_dash_b_fail", [](testing &t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("b"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escaped_dash_b_fail", true, result.fail()); }); });
t.test("optional", [](testing & t) { // Full match with optional part present t.test("optional_present", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); });
auto ctx = common_peg_parse_context("hello world"); auto result = parser.parse(ctx); t.assert_equal("optional_present", true, result.success()); t.assert_equal("optional_present_end", 11u, result.end); });
// Full match with optional part absent t.test("optional_absent", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); });
auto ctx = common_peg_parse_context("hello"); auto result = parser.parse(ctx); t.assert_equal("optional_absent", true, result.success()); t.assert_equal("optional_absent_end", 5u, result.end); });
// Partial match - waiting for more input to determine if optional matches t.test("partial_match_need_more", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); });
auto ctx = common_peg_parse_context("hello ", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("partial_match_need_more", true, result.need_more_input()); }); });
t.test("partial parsing", [](testing & t) { // Literals - Basic Success t.test("literal_success", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("hello"); result = parser.parse(ctx); t.assert_equal("literal_success", true, result.success()); });
// Char Classes - Basic Lowercase Success t.test("char_class_lowercase_success", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("a"); result = parser.parse(ctx); t.assert_equal("char_class_lowercase_success", true, result.success()); });
// Char Classes - Uppercase Fail t.test("char_class_uppercase_fail", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("A"); result = parser.parse(ctx); t.assert_equal("char_class_uppercase_fail", true, result.fail()); });
// Char Classes with Dash - Lowercase Success t.test("char_class_with_dash_lowercase", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("f"); result = parser.parse(ctx); t.assert_equal("char_class_with_dash_lowercase", true, result.success()); });
// Char Classes with Dash - Literal Dash Success t.test("char_class_with_dash_literal_dash", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("-"); result = parser.parse(ctx); t.assert_equal("char_class_with_dash_literal_dash", true, result.success()); });
// Char Classes with Dash - Uppercase Fail t.test("char_class_with_dash_uppercase_fail", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
common_peg_parse_context ctx; common_peg_parse_result result;
ctx = common_peg_parse_context("A"); result = parser.parse(ctx); t.assert_equal("char_class_with_dash_uppercase_fail", true, result.fail()); });
// Sequences - Partial Match 1 t.test("sequence_partial_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<thi", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("sequence_partial_match_1", true, result.need_more_input()); });
// Sequences - Partial Match 2 t.test("sequence_partial_match_2", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("begin") + p.literal("end"); });
auto ctx = common_peg_parse_context("begin", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("sequence_partial_match_2", true, result.need_more_input()); });
// Sequences - Partial Match 3 t.test("sequence_partial_match_3", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<think></", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("sequence_partial_match_3", true, result.need_more_input()); });
// Sequences - Full Match t.test("sequence_full_match", [&](testing & t) { auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.literal("world"); });
auto ctx = common_peg_parse_context("helloworld"); auto result = common_chat_combinator_parser.parse(ctx); t.assert_equal("sequence_full_match", true, result.success()); });
// Sequences - No Match t.test("sequence_no_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<think>I am common_chat_combinator_parser", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("sequence_no_match", true, result.fail()); });
// Choices - Partial Match 1 t.test("choices_partial_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); });
auto ctx = common_peg_parse_context("opt", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("choices_partial_match_1", true, result.need_more_input()); });
// Choices - Partial Match 2 t.test("choices_partial_match_2", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); });
auto ctx = common_peg_parse_context("choice", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("choices_partial_match_2", true, result.need_more_input()); });
// Choices - Full Match 1 t.test("choices_full_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); });
auto ctx = common_peg_parse_context("first"); auto result = parser.parse(ctx); t.assert_equal("choices_full_match_1", true, result.success()); });
// Choices - Full Match 2 t.test("choices_full_match_2", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); });
auto ctx = common_peg_parse_context("beta"); auto result = parser.parse(ctx); t.assert_equal("choices_full_match_2", true, result.success()); });
// Choices - No Match t.test("choices_no_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); });
auto ctx = common_peg_parse_context("best"); auto result = parser.parse(ctx); t.assert_equal("choices_no_match", true, result.fail()); });
// Zero or More - Partial Match 1 t.test("zero_or_more_partial_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); });
auto ctx = common_peg_parse_context("a", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input()); });
// Zero or More - Partial Match 2 t.test("zero_or_more_partial_match_2", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); });
auto ctx = common_peg_parse_context("xyx", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input()); });
// Zero or More - Full Match t.test("zero_or_more_full_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); });
auto ctx = common_peg_parse_context("test"); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_full_match", true, result.success()); });
// One or More - Partial Match 1 t.test("one_or_more_partial_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); });
auto ctx = common_peg_parse_context("rep", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input()); });
// One or More - Partial Match 2 t.test("one_or_more_partial_match_2", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); });
auto ctx = common_peg_parse_context("aba", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = parser.parse(ctx); t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input()); });
// One or More - Full Match t.test("one_or_more_full_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); });
auto ctx = common_peg_parse_context("single"); auto result = parser.parse(ctx); t.assert_equal("one_or_more_full_match", true, result.success()); });
// One or More - No Match t.test("one_or_more_no_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); });
auto ctx = common_peg_parse_context("success"); auto result = parser.parse(ctx); t.assert_equal("one_or_more_no_match", true, result.fail()); }); });
t.test("recursive rules", [](testing &t) { // Test simple number t.test("simple_number", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); });
common_peg_parse_context ctx("1"); auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); });
// Test simple list t.test("simple_list", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); });
common_peg_parse_context ctx("[1]"); auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); });
// Test nested list t.test("nested_list", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); });
common_peg_parse_context ctx("[[2]]"); auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); });
// Test deeply nested list t.test("deeply_nested_list", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); });
common_peg_parse_context ctx("[[[3]]]"); auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); });
// Test need_more_input match t.test("need_more_input_match", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); });
common_peg_parse_context ctx("[[", COMMON_PEG_PARSE_FLAG_LENIENT); auto result = value_parser.parse(ctx);
t.assert_equal("result_is_need_more_input", true, result.need_more_input()); });
// Test no match t.test("no_match", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); });
common_peg_parse_context ctx("[a]"); auto result = value_parser.parse(ctx);
t.assert_equal("result_is_fail", true, result.fail()); });
// Test markers t.test("marker", [](testing &t) { auto bracket_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.marker(); });
common_peg_parse_context ctx_square("[marker]"); common_peg_parse_context ctx_sharp("<marker>");
auto result_square = bracket_parser.parse(ctx_square); auto result_sharp = bracket_parser.parse(ctx_sharp);
t.assert_true("result_square_is_success", result_square.success()); t.assert_true("result_sharp_is_success", result_sharp.success()); }); });}