Something went wrong. Try again.
llama.cpp fork for Ternary Bonsai 2 27B: MTP speculation, suffix cache reuse (SCR), q8 KV, two-slot fairness, fused DeltaNet kernels
Something went wrong. Try again.
13 kB · 318 lines
C++
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319#include "tests.h"
void test_python_dict_parser(testing &t) { // Test parsing a simple Python dict object with single quotes t.test("simple Python dict object parsing", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test', 'value': 42, 'flag': True}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test parsing a Python array with mixed types t.test("Python array with mixed types", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "[1, 'hello', True, None, 3.14]"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test parsing nested Python dict with objects and arrays t.test("nested Python dict with objects and arrays", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'users': [{'id': 1, 'name': 'Alice'}, {'id': 2, 'name': 'Bob'}], 'count': 2, 'metadata': {'version': '1.0', 'tags': ['admin', 'user']}}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test parsing Python dict with escaped single quotes t.test("Python dict with escaped single quotes", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'message': 'It\\'s working!'}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test parsing Python dict with double quotes inside single quotes t.test("Python dict with double quotes inside single quotes", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'quote': 'He said \"Hello\"'}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test the example from the requirements t.test("complex Python dict example from requirements", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{ 'obj' : { 'something': 1, 'other \"something\"' : 'foo\\'s bar' } }"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test need_more_input() parsing - incomplete object t.test("need_more_input() parsing - incomplete object", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test', 'value': "; common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("result_is_need_more_input", true, result.need_more_input()); });
// Test need_more_input() parsing - incomplete single-quoted string t.test("need_more_input() parsing - incomplete single-quoted string", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test"; common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("result_is_need_more_input", true, result.need_more_input()); });
// Test unicode in Python dict strings t.test("unicode in Python dict strings", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'message': 'Hello, 世界!'}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test Python dict with unicode escapes t.test("Python dict with unicode escapes", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'unicode': 'Hello\\u0041'}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test that Python parser accepts double-quoted strings too t.test("Python parser accepts double-quoted strings", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{\"name\": \"test\"}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test Python parser with mixed quote styles t.test("Python parser with mixed quote styles", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{\"name\": 'test', 'value': \"hello\"}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success()); t.assert_equal("result_end", input.size(), result.end); });
// Test Python True/False/None t.test("Python True/False/None", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
t.test("True", [&](testing &t) { std::string input = "True"; common_peg_parse_context ctx(input); auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("False", [&](testing &t) { std::string input = "False"; common_peg_parse_context ctx(input); auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("None", [&](testing &t) { std::string input = "None"; common_peg_parse_context ctx(input); auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("rejects JSON-style true/false/null", [&](testing &t) { for (const auto & kw : {"true", "false", "null"}) { std::string input = kw; common_peg_parse_context ctx(input); auto result = parser.parse(ctx); t.assert_true(std::string("rejects ") + kw, result.fail()); } }); });
// Test single-quoted string content parser directly t.test("single-quoted string content parser", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.sequence({ p.literal("'"), p.string_content('\''), p.literal("'"), p.space() }); });
t.test("simple string", [&](testing &t) { std::string input = "'hello'"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("string with escaped single quote", [&](testing &t) { std::string input = "'it\\'s'"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("string with double quotes", [&](testing &t) { std::string input = "'say \"hello\"'"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("incomplete string", [&](testing &t) { std::string input = "'hello"; common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx); t.assert_true("need_more_input", result.need_more_input()); }); });
// Test json() with pre-registered flexible json-string rule (python dict support) t.test("json() parser with flexible json-string rule", [](testing &t) { t.test("json() rejects single quotes by default", [&](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
std::string input = "{'name': 'test'}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx); t.assert_true("fail", result.fail()); });
t.test("json() accepts single quotes with pre-registered flexible json-string rule", [&](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { // Pre-register json-string rule with both quote styles p.rule("json-string", [&]() { return p.choice({ p.double_quoted_string(), p.single_quoted_string() }); }); return p.json(); });
std::string input = "{'name': 'test'}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("json() still accepts double quotes with flexible json-string rule", [&](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("json-string", [&]() { return p.choice({ p.double_quoted_string(), p.single_quoted_string() }); }); return p.json(); });
std::string input = "{\"name\": \"test\"}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("json() accepts mixed quote styles with flexible json-string rule", [&](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("json-string", [&]() { return p.choice({ p.double_quoted_string(), p.single_quoted_string() }); }); return p.json(); });
std::string input = "{\"name\": 'test', 'value': \"hello\"}"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); });
t.test("complex nested structure with flexible json-string rule", [&](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("json-string", [&]() { return p.choice({ p.double_quoted_string(), p.single_quoted_string() }); }); return p.json(); });
std::string input = "{ 'obj' : { 'something': 1, 'other \"something\"' : 'foo\\'s bar' } }"; common_peg_parse_context ctx(input);
auto result = parser.parse(ctx); t.assert_true("success", result.success()); t.assert_equal("end", input.size(), result.end); }); });}