#include "chat-peg-parser.h" #include "chat.h" #include "common.h" #include "json-schema-to-grammar.h" #include "peg-parser.h" #include "testing.h" #include "peg-parser/simple-tokenize.h" #include #include #include #include #include "json.h" using json = common_json; static json create_tools(); static void test_example_native(testing & t); static void test_example_qwen3_coder(testing & t); static void test_example_qwen3_non_coder(testing & t); static void test_command7_parser_compare(testing & t); static void test_prefix_tool_names(testing & t); static void test_tagged_peg_parser(testing & t); static void test_permute(testing & t); int main(int argc, char * argv[]) { testing t(std::cout); if (argc >= 2) { t.set_filter(argv[1]); } const char * verbose = getenv("LLAMA_TEST_VERBOSE"); if (verbose) { t.verbose = std::string(verbose) == "1"; } t.test("native", test_example_native); t.test("qwen3 coder", test_example_qwen3_coder); t.test("qwen3 non-coder", test_example_qwen3_non_coder); t.test("comparison", test_command7_parser_compare); t.test("prefix tool names", test_prefix_tool_names); t.test("tagged peg parser", test_tagged_peg_parser); t.test("permute", test_permute); return t.summary(); } static json create_tools() { json tools = json::array(); json tool_weather = { { "type", "function" }, { "function", { { "name", "get_current_weather" }, { "description", "Get the current weather in a given location" }, { "parameters", { { "type", "object" }, { "properties", { { "location", { { "type", "string" }, { "description", "The city and state, e.g. San Francisco, CA" } } }, { "unit", { { "type", "string" }, { "enum", json::array({ "celsius", "fahrenheit" }) }, { "description", "The temperature unit to use. Infer this from the users location." } } } } }, { "required", json::array({ "location", "unit" }) }, } }, } } }; tools.push_back(tool_weather); json tool_forecast = { { "type", "function" }, { "function", { { "name", "get_forecast" }, { "description", "Get the weather forecast for a given location" }, { "parameters", { { "type", "object" }, { "properties", { { "location", { { "type", "string" }, { "description", "The city and state, e.g. San Francisco, CA" } } }, { "unit", { { "type", "string" }, { "enum", json::array({ "celsius", "fahrenheit" }) }, { "description", "The temperature unit to use. Infer this from the users location." } } }, { "days", { { "type", "integer" }, { "description", "Number of days to forecast (1-10)" }, { "minimum", 1 }, { "maximum", 10 } } } } }, { "required", json::array({ "location", "unit" }) }, } }, } } }; tools.push_back(tool_forecast); json tool_search = { { "type", "function" }, { "function", { { "name", "search_knowledge_base" }, { "description", "Search the internal technical documentation knowledge base." }, { "parameters", { { "type", "object" }, { "properties", { { "query", { { "type", "string" }, { "description", "The search query string." } } }, { "max_results", { { "type", "integer" }, { "description", "The maximum number of results to return." }, { "default", 5 } } }, { "category", { { "type", "string" }, { "enum", json::array({ "api", "troubleshooting", "billing", "general" }) }, { "description", "Filter search by specific category." } } } } }, { "required", json::array({ "query", "category" }) }, { "additionalProperties", false } } }, { "strict", true } } } }; tools.push_back(tool_search); return tools; } struct tool_argument { std::string name; std::string type; bool is_required; json schema; }; struct tool_definition { std::string name; std::vector arguments; json schema; }; // Test fictitious model output that emits arguments as JSON. static void test_example_native(testing & t) { struct test_case { // Parameters std::string name; json tools; common_chat_tool_choice tool_choice; common_reasoning_format reasoning_format; json json_schema; bool parallel_tool_calls; std::string generation_prompt; std::string input; // Expect std::string expect_reasoning; std::string expect_content; std::vector expect_tool_calls; }; auto build_parser = [](const test_case & tc) { return build_chat_peg_parser([&](common_chat_peg_builder & p) { auto reasoning_in_content = (tc.reasoning_format == COMMON_REASONING_FORMAT_NONE); // Always use optional TAG_BASED pattern; generation_prompt is prepended to input auto reasoning = p.optional("" + p.reasoning(p.until("")) + "" + p.space()); // tool calling parser if (tc.tools.is_array() && !tc.tools.empty()) { auto tool_call = p.standard_json_tools("[", "]", tc.tools, tc.parallel_tool_calls, tc.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED); return p.sequence({ (reasoning_in_content ? p.eps() : reasoning), p.content(p.until("")), p.optional(p.space() + tool_call), p.space(), p.end() }); } // response_format parser if (tc.json_schema.is_object() && !tc.json_schema.empty()) { return p.sequence({ (reasoning_in_content ? p.eps() : reasoning), p.content(p.schema(p.json(), "response-output", tc.json_schema)), p.space(), p.end() }); } // Content-only parser return p.sequence({ (reasoning_in_content ? p.eps() : reasoning), p.content(p.rest()), p.end() }); }); }; std::vector test_cases = std::vector{ { /* .name = */ "content with reasoning (no generation_prompt)", /* .tools = */ {}, /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_NONE, /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, /* .json_schema = */ {}, /* .parallel_tool_calls = */ false, /* .generation_prompt = */ "", /* .input = */ ("The user said hello, I must say hello back\nHello"), /* .expect_reasoning = */ "The user said hello, I must say hello back", /* .expect_content = */ "Hello", /* .expect_tool_calls = */ {}, }, { /* .name = */ "content without reasoning (no generation_prompt)", /* .tools = */ {}, /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_NONE, /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, /* .json_schema = */ {}, /* .parallel_tool_calls = */ false, /* .generation_prompt = */ "", /* .input = */ ("Hello"), /* .expect_reasoning = */ "", /* .expect_content = */ "Hello", /* .expect_tool_calls = */ {}, }, { /* .name = */ "content with reasoning_format = none (tags appear in content)", /* .tools = */ {}, /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_NONE, /* .reasoning_format = */ COMMON_REASONING_FORMAT_NONE, /* .json_schema = */ {}, /* .parallel_tool_calls = */ false, /* .generation_prompt = */ "", /* .input = */ ("The user said hello, I must say hello back\nHello"), /* .expect_reasoning = */ "", /* .expect_content = */ "The user said hello, I must say hello back\nHello", /* .expect_tool_calls = */ {}, }, { /* .name = */ "content with reasoning generation_prompt", /* .tools = */ {}, /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_NONE, /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, /* .json_schema = */ {}, /* .parallel_tool_calls = */ false, /* .generation_prompt = */ "", /* .input = */ ("The user said hello, I must say hello back\nHello"), /* .expect_reasoning = */ "The user said hello, I must say hello back", /* .expect_content = */ "Hello", /* .expect_tool_calls = */ {}, }, { /* .name = */ "content with reasoning generation_prompt and reasoning_format = none", /* .tools = */ {}, /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_NONE, /* .reasoning_format = */ COMMON_REASONING_FORMAT_NONE, /* .json_schema = */ {}, /* .parallel_tool_calls = */ false, /* .generation_prompt = */ "", /* .input = */ ("The user said hello, I must say hello back\nHello"), /* .expect_reasoning = */ "", /* .expect_content = */ "The user said hello, I must say hello back\nHello", /* .expect_tool_calls = */ {}, }, { /* .name = */ "content with closed reasoning generation_prompt (empty reasoning discarded)", /* .tools = */ {}, /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_NONE, /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, /* .json_schema = */ {}, /* .parallel_tool_calls = */ false, /* .generation_prompt = */ "", /* .input = */ ("Hello"), /* .expect_reasoning = */ "", /* .expect_content = */ "Hello", /* .expect_tool_calls = */ {}, }, { /* .name = */ "tools with reasoning generation_prompt", /* .tools = */ create_tools(), /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_AUTO, /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, /* .json_schema = */ {}, /* .parallel_tool_calls = */ false, /* .generation_prompt = */ "", /* .input = */ ("I must get the weather in New York\n" "[" R"({"name": "get_current_weather", "arguments": {"location": "New York City, NY", "unit": "fahrenheit"}})" "]"), /* .expect_reasoning = */ "I must get the weather in New York", /* .expect_content = */ "", /* .expect_tool_calls = */ { { /* .name = */ "get_current_weather", /* .arguments = */ R"({"location": "New York City, NY", "unit": "fahrenheit"})", /* .id = */ "", } }, }, { /* .name = */ "parallel tools with reasoning generation_prompt", /* .tools = */ create_tools(), /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_AUTO, /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, /* .json_schema = */ {}, /* .parallel_tool_calls = */ true, /* .generation_prompt = */ "", /* .input = */ ("I must get the weather in New York and San Francisco and a 3 day forecast of each.\nLet me " "search that for you." "[" R"({"name": "get_current_weather", "arguments": {"location": "New York City, NY", "unit": "fahrenheit"}})" ", " R"({"name": "get_current_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}})" ", " R"({"name": "get_forecast", "arguments": {"location": "New York City, NY", "unit": "fahrenheit", "days": 3}})" ", " R"({"name": "get_forecast", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit", "days": 3}})" "]"), /* .expect_reasoning = */ "I must get the weather in New York and San Francisco and a 3 day forecast of each.", /* .expect_content = */ "Let me search that for you.", /* .expect_tool_calls = */ { { /* .name = */ "get_current_weather", /* .arguments = */ R"({"location": "New York City, NY", "unit": "fahrenheit"})", /* .id = */ "", }, { /* .name = */ "get_current_weather", /* .arguments = */ R"({"location": "San Francisco, CA", "unit": "fahrenheit"})", /* .id = */ "", }, { /* .name = */ "get_forecast", /* .arguments = */ R"({"location": "New York City, NY", "unit": "fahrenheit", "days": 3})", /* .id = */ "", }, { /* .name = */ "get_forecast", /* .arguments = */ R"({"location": "San Francisco, CA", "unit": "fahrenheit", "days": 3})", /* .id = */ "", } }, }, { /* .name = */ "response_format with reasoning generation_prompt", /* .tools = */ {}, /* .tool_choice = */ COMMON_CHAT_TOOL_CHOICE_NONE, /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, /* .json_schema = */ { { "type", "object" }, { "properties", { { "invoice_number", { { "type", "string" } } }, { "amount", { { "type", "number" } } }, { "due_date", { { "type", "string" } } } } }, { "required", json::array({ "invoice_number", "amount", "due_date" }) } }, /* .parallel_tool_calls = */ false, /* .generation_prompt = */ "", /* .input = */ ("I must produce the invoice in the requested format\n" R"({"invoice_number": "INV-2025-001", "amount": 1250.50, "due_date": "2025-12-31"})"), /* .expect_reasoning = */ "I must produce the invoice in the requested format", /* .expect_content = */ R"({"invoice_number": "INV-2025-001", "amount": 1250.50, "due_date": "2025-12-31"})", /* .expect_tool_calls = */ {}, }, }; for (const auto & tc : test_cases) { t.test(tc.name, [&](testing & t) { auto parser = build_parser(tc); auto lazy = !tc.tools.empty() && tc.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; auto grammar = build_grammar([&](const common_grammar_builder & builder) { for (const auto & def : tc.tools) { auto function = def.at("function"); auto parameters = function.at("parameters"); builder.resolve_refs(parameters); }; parser.build_grammar(builder, lazy); }); t.log("Grammar:"); for (const auto & line : string_split(grammar, "\n")) { t.log(line); } std::string effective_input = tc.generation_prompt + tc.input; common_peg_parse_context ctx(effective_input); auto result = parser.parse(ctx); t.assert_true("success", result.success()); common_chat_msg msg; auto mapper = common_chat_peg_mapper(msg); mapper.from_ast(ctx.ast, result); t.assert_equal("content equal", tc.expect_content, msg.content); t.assert_equal("reasoning equal", tc.expect_reasoning, msg.reasoning_content); t.assert_equal("number of tool calls", tc.expect_tool_calls.size(), msg.tool_calls.size()); for (auto i = 0u; i < std::min(tc.expect_tool_calls.size(), msg.tool_calls.size()); i++) { t.assert_equal("tool name", tc.expect_tool_calls[i].name, msg.tool_calls[i].name); t.assert_equal("tool args", tc.expect_tool_calls[i].arguments, msg.tool_calls[i].arguments); } }); } } static void test_example_qwen3_coder(testing & t) { auto tools = create_tools(); auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto content = p.rule("content", p.content(p.until(""))); std::vector tool_parsers; for (const auto & def : tools) { auto function = def.at("function"); std::string name = function.at("name"); auto parameters = function.at("parameters"); auto properties = parameters.at("properties"); std::set required_properties; if (function.contains("required")) { required_properties = function.at("required").get>(); } std::vector arg_parsers; for (const auto & [param_name, param_schema] : properties.items()) { bool is_required = required_properties.find(param_name) != required_properties.end(); auto type = param_schema.value("type", "object"); auto arg = p.tool_arg( p.sequence({ p.tool_arg_open(""), (type == "string" ? p.tool_arg_string_value(p.schema( p.until_one_of({ "\n\n" }), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, true)) : p.tool_arg_json_value(p.schema( p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema))), p.tool_arg_close("\n" + p.peek(p.literal(""))) })); arg_parsers.push_back(is_required ? p.rule("tool-" + name + "-arg-" + param_name, arg) : p.optional(p.rule("tool-" + name + "-arg-" + param_name, arg))); } tool_parsers.push_back(p.rule("tool-" + name, p.tool_open("") << p.sequence(arg_parsers) << p.tool_close(p.literal("")))); }; auto tool_call = p.trigger_rule("tool-call", "" << p.choice(tool_parsers) << ""); return content + p.zero_or_more(p.space() + tool_call) + p.end(); }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { for (const auto & def : tools) { auto function = def.at("function"); auto parameters = function.at("parameters"); builder.resolve_refs(parameters); }; parser.build_grammar(builder); }); t.log("Grammar:"); for (const auto & line : string_split(grammar, "\n")) { t.log(line); } t.test("incremental parsing", [&](testing & t) { std::string input = "Let me search the knowledge base for cat pictures." "\n" "\n" "cat pictures\n" "general\n" "\n" ""; std::vector tokens = simple_tokenize(input); common_chat_msg prev; for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE); auto result = parser.parse(ctx); if (!t.assert_equal("not fail", false, result.fail())) { t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); } common_chat_msg msg; auto mapper = common_chat_peg_mapper(msg); mapper.from_ast(ctx.ast, result); //t.log("Input: " + input); t.log("==========================================="); t.log("Iteration " + std::to_string(in.size())); t.log("Reasoning: " + msg.reasoning_content); t.log("Content : " + msg.content); for (const auto & tc : msg.tool_calls) { t.log("Tool name: " + tc.name); t.log("Tool args: " + tc.arguments); } try { // This shouldn't emit any runtime errors auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); } catch (const std::exception & e) { t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); t.assert_true(std::string("failed with ") + e.what(), false); } prev = msg; } }); } static void test_example_qwen3_non_coder(testing & t) { auto tools = create_tools(); auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { // tool calling parser using standard JSON format auto tool_call = p.standard_json_tools("", "", tools, true, false); return p.sequence({ p.content(p.until("")), p.optional(p.space() + tool_call), p.end() }); }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { for (const auto & def : tools) { auto function = def.at("function"); auto parameters = function.at("parameters"); builder.resolve_refs(parameters); }; parser.build_grammar(builder); }); t.log("Grammar:"); for (const auto & line : string_split(grammar, "\n")) { t.log(line); } t.test("tool call parsing", [&](testing & t) { std::string input = "I need to get the weather.\n" "" "{\"name\": \"get_current_weather\", \"arguments\": {\"location\": \"New York City, NY\", \"unit\": " "\"fahrenheit\"}}" ""; common_peg_parse_context ctx(input); auto result = parser.parse(ctx); t.assert_true("success", result.success()); common_chat_msg msg; auto mapper = common_chat_peg_mapper(msg); mapper.from_ast(ctx.ast, result); t.assert_equal("content", "I need to get the weather.\n", msg.content); t.assert_equal("reasoning", "", msg.reasoning_content); t.assert_equal("tool calls count", 1u, msg.tool_calls.size()); if (!msg.tool_calls.empty()) { t.assert_equal("tool name", "get_current_weather", msg.tool_calls[0].name); t.assert_equal("tool args", "{\"location\": \"New York City, NY\", \"unit\": \"fahrenheit\"}", msg.tool_calls[0].arguments); } }); t.test("incremental parsing", [&](testing & t) { std::string input = "I need to get the weather.\n" "" "{\"name\": \"get_current_weather\", \"arguments\": {\"location\": \"New York City, NY\", \"unit\": " "\"fahrenheit\"}}" ""; std::vector tokens = simple_tokenize(input); common_chat_msg prev; for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE); auto result = parser.parse(ctx); if (!t.assert_equal("not fail", false, result.fail())) { t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); } common_chat_msg msg; auto mapper = common_chat_peg_mapper(msg); mapper.from_ast(ctx.ast, result); //t.log("Input: " + input); t.log("==========================================="); t.log("Iteration " + std::to_string(in.size())); t.log("Reasoning: " + msg.reasoning_content); t.log("Content : " + msg.content); for (const auto & tc : msg.tool_calls) { t.log("Tool name: " + tc.name); t.log("Tool args: " + tc.arguments); } try { // This shouldn't emit any runtime errors auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); } catch (const std::exception & e) { t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); t.assert_true(std::string("failed with ") + e.what(), false); } prev = msg; } }); } void test_command7_parser_compare(testing & t) { auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { auto thinking = p.reasoning_block("<|START_THINKING|>" << p.reasoning(p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); auto response = "<|START_RESPONSE|>" << p.content(p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"; auto tool_call_id = p.atomic("\"tool_call_id\"" << (":" << ("\"" + p.tool_id(p.string_content('"')) + "\""))); auto tool_call_name = p.atomic("\"tool_name\"" << (":" << ("\"" + p.tool_name(p.string_content('"')) + "\""))); auto tool_call_args = "\"parameters\"" << (":" << p.tool_args(p.json())); auto tool_call_fields = p.rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); auto tool_call = p.rule("tool-call", p.tool(p.tool_open(p.literal("{")) << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << p.tool_close(p.literal("}")))); auto tool_calls = p.rule( "tool-calls", "<|START_ACTION|>" << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") << "<|END_ACTION|>"); return p.optional(thinking) << (tool_calls | response) + p.end(); }); auto test_current = [&](const common_peg_arena & p, const std::string & input, bool is_partial, bool print_results) { common_peg_parse_context ctx(input, is_partial ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE); auto result = p.parse(ctx); common_chat_msg msg; auto mapper = common_chat_peg_mapper(msg); mapper.from_ast(ctx.ast, result); if (print_results) { std::cout << "== Parsed (new) ==\n"; std::cout << "=== Reasoning ===\n"; std::cout << msg.reasoning_content << "\n"; std::cout << "\n\n=== Content ===\n"; std::cout << msg.content << "\n"; std::cout << "\n\n=== Tool Calls ===\n"; for (const auto & tc : msg.tool_calls) { std::cout << "id: " << tc.id << "\n"; std::cout << "name: " << tc.name << "\n"; std::cout << "args: " << tc.arguments << "\n"; } } }; std::string reasoning = "To plan an effective trip to Japan that includes both historical sites and modern attractions within a " "budget of $4000 for a two-week stay, we need to:\n\n" "1. Identify key historical sites and modern attractions in Japan.\n" "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" "3. Determine the best modes of transportation for getting around Japan.\n" "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without " "overspending.\n" "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " "to attractions."; std::vector> tool_calls = { { "call_0", "plan_trip", common_json::parse(R"({ "destination": "Japan", "duration": 14, "budget": 4000, "interests": ["historical sites", "modern attractions"], "accommodation_preferences": "affordable", "transportation_preferences": "efficient", "meal_preferences": "local cuisine" })") } }; std::vector tokens; // Build tokens if (!reasoning.empty()) { auto tokenized = simple_tokenize(reasoning); tokens.emplace_back("<|START_THINKING|>"); tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); tokens.emplace_back("<|END_THINKING|>"); } if (!tool_calls.empty()) { tokens.emplace_back("<|START_ACTION|>"); auto json = common_json::array(); for (const auto & tc : tool_calls) { auto tc_json = common_json::object(); tc_json["tool_call_id"] = std::get<0>(tc); tc_json["tool_name"] = std::get<1>(tc); tc_json["parameters"] = std::get<2>(tc); json.push_back(tc_json); } auto tokenized = simple_tokenize(json.dump(-1)); tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); tokens.emplace_back("<|END_ACTION|>"); } std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); t.test("current_parse", [&](testing & /* t */) { test_current(parser, input, false, false); }); t.bench("current_parse_benchmark complete", [&]() { test_current(parser, input, false, false); }, 100); t.bench( "current_parse_benchmark incremental", [&]() { std::string in; for (auto i = 0u; i < tokens.size(); i++) { in += tokens[i]; test_current(parser, in, i + 1 < tokens.size(), false); } }, 20); } // Test that tool names that are proper prefixes of other tool names don't cause // premature matching during incremental parsing. // For example, "special_function" should not match when parsing "special_function_with_opt". static void test_prefix_tool_names(testing & t) { // Create tools where one name is a proper prefix of another json tools = json::array(); json tool_short = { { "type", "function" }, { "function", { { "name", "special_function" }, { "description", "A special function" }, { "parameters", { { "type", "object" }, { "properties", { { "arg1", { { "type", "integer" } } }, } }, { "required", json::array({ "arg1" }) }, } }, } } }; tools.push_back(tool_short); json tool_long = { { "type", "function" }, { "function", { { "name", "special_function_with_opt" }, { "description", "A special function with optional params" }, { "parameters", { { "type", "object" }, { "properties", { { "arg1", { { "type", "integer" } } }, { "arg2", { { "type", "integer" } } }, } }, { "required", json::array({ "arg1" }) }, } }, } } }; tools.push_back(tool_long); // Use standard_constructed_tools which had the prefix matching bug std::map markers = { { "tool_call_start_marker", "" }, { "tool_call_end_marker", "" }, { "function_opener", "" }, { "function_name_suffix", ">" }, { "parameter_key_prefix", "" }, { "parameter_closer", "" }, }; auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto content = p.rule("content", p.content(p.until(""))); auto tool_call = p.standard_constructed_tools(markers, tools, false, false); return content + p.zero_or_more(p.space() + tool_call) + p.end(); }); // Test parsing the long tool name - this should NOT trigger the short tool name t.test("parse long tool name", [&](testing & t) { std::string input = "Let me call the function." "" "" "42" "" ""; common_peg_parse_context ctx(input); auto result = parser.parse(ctx); t.assert_true("success", result.success()); common_chat_msg msg; auto mapper = common_chat_peg_mapper(msg); mapper.from_ast(ctx.ast, result); t.assert_equal("content", "Let me call the function.", msg.content); t.assert_equal("tool calls count", 1u, msg.tool_calls.size()); if (!msg.tool_calls.empty()) { t.assert_equal("tool name", "special_function_with_opt", msg.tool_calls[0].name); } }); // Test incremental parsing - the key test case // This ensures that when incrementally parsing "special_function_with_opt", // we don't prematurely emit "special_function" as a tool call t.test("incremental parse long tool name", [&](testing & t) { std::string input = "Let me call the function." "" "" "42" "" ""; std::vector tokens = simple_tokenize(input); common_chat_msg prev; for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE); auto result = parser.parse(ctx); if (!t.assert_equal("not fail", false, result.fail())) { t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); return; } common_chat_msg msg; auto mapper = common_chat_peg_mapper(msg); mapper.from_ast(ctx.ast, result); // The critical check: during incremental parsing, we should never // see "special_function" as the tool name when parsing "special_function_with_opt" for (const auto & tc : msg.tool_calls) { if (!t.assert_equal("tool name should not be short prefix", false, tc.name == "special_function")) { t.log("Premature tool name match at input: " + in); return; } } try { auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); } catch (const std::exception & e) { t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); t.assert_true(std::string("diff failed with ") + e.what(), false); return; } prev = msg; } // Final check: the complete parse should have the correct tool name t.assert_equal("final tool calls count", 1u, prev.tool_calls.size()); if (!prev.tool_calls.empty()) { t.assert_equal("final tool name", "special_function_with_opt", prev.tool_calls[0].name); } }); // Test parsing the short tool name still works t.test("parse short tool name", [&](testing & t) { std::string input = "Let me call the function." "" "" "42" "" ""; common_peg_parse_context ctx(input); auto result = parser.parse(ctx); t.assert_true("success", result.success()); common_chat_msg msg; auto mapper = common_chat_peg_mapper(msg); mapper.from_ast(ctx.ast, result); t.assert_equal("content", "Let me call the function.", msg.content); t.assert_equal("tool calls count", 1u, msg.tool_calls.size()); if (!msg.tool_calls.empty()) { t.assert_equal("tool name", "special_function", msg.tool_calls[0].name); } }); } static void test_tagged_peg_parser(testing & t) { t.test("basic tag extraction", [&](testing & t) { auto parser = build_tagged_peg_parser([](common_peg_parser_builder & p) { return p.tag("greeting", p.until(" ")) + " " + p.tag("name", p.rest()) + p.end(); }); auto result = parser.parse_and_extract("Hello World"); t.assert_true("success", result.result.success()); t.assert_equal("greeting tag", "Hello", result.tags.at("greeting")); t.assert_equal("name tag", "World", result.tags.at("name")); }); t.test("duplicate tags overwrite", [&](testing & t) { auto parser = build_tagged_peg_parser([](common_peg_parser_builder & p) { return p.tag("item", p.until(",")) + "," + p.tag("item", p.rest()) + p.end(); }); auto result = parser.parse_and_extract("first,second"); t.assert_true("success", result.result.success()); t.assert_equal("item tag", "second", result.tags.at("item")); }); t.test("no tags extracted", [&](testing & t) { auto parser = build_tagged_peg_parser([](common_peg_parser_builder & p) { return p.rest() + p.end(); }); auto result = parser.parse_and_extract("Hello"); t.assert_true("success", result.result.success()); t.assert_equal("empty tags", 0u, result.tags.size()); }); t.test("structured extraction", [&](testing & t) { auto parser = build_tagged_peg_parser([](common_peg_parser_builder & p) { auto header = p.tag("header", p.until("\n")); auto body = p.tag("body", p.rest()); return header + "\n" + body + p.end(); }); auto result = parser.parse_and_extract("Title\nBody content here"); t.assert_true("success", result.result.success()); t.assert_equal("header", "Title", result.tags.at("header")); t.assert_equal("body", "Body content here", result.tags.at("body")); }); t.test("partial parse", [&](testing & t) { auto parser = build_tagged_peg_parser([](common_peg_parser_builder & p) { return p.tag("prefix", p.until(":")) + ":" + p.tag("value", p.rest()) + p.end(); }); auto result = parser.parse_and_extract("key:val", COMMON_PEG_PARSE_FLAG_LENIENT); t.assert_true("not fail", !result.result.fail()); t.assert_equal("prefix tag", "key", result.tags.at("prefix")); t.assert_equal("value tag", "val", result.tags.at("value")); }); t.test("find in the middle", [&](testing & t) { auto parser = build_tagged_peg_parser([](common_peg_parser_builder & p) { return p.choice({ p.literal("{"), p.literal(":") }) + p.space() + p.literal("\"") + p.atomic(p.literal("fun_name")); }); std::string tpl = "This is a very long jinja template string. We have tools. We will try to call them now: { \"fun_name\" : { \"arg\" : 1 }"; auto result = parser.parse_anywhere_and_extract(tpl); t.assert_true("success", result.result.success()); }); t.test("fail find in the middle", [&](testing & t) { auto parser = build_tagged_peg_parser([](common_peg_parser_builder & p) { return p.choice({ p.literal("{"), p.literal(":") }) + p.space() + p.literal("\"") + p.atomic(p.literal("fun_name")); }); std::string tpl = "This is a very long jinja template string. We have tools. We will try to call them now: 1"; auto result = parser.parse_anywhere_and_extract(tpl); t.assert_true("failure", result.result.fail()); }); t.test("find function tag with name", [&](testing &t) { std::string haystack = "\n\n\n\nXXXX\n\n\nYYYY\n\n\n\n"; auto parser = build_tagged_peg_parser([](common_peg_parser_builder & p) { std::string needle = "foofoo"; return p.tag("fun_marker", p.choice({ p.tag("fun_pre", p.literal("<") + p.until_one_of({ ">", needle })) + p.literal(needle) + p.tag("fun_post", p.negate(p.space() + p.literal("<")) + p.until(">") + p.literal(">")) + p.space(), p.tag("fun_pre", p.literal("[") + p.until_one_of({ "]", needle })) + p.literal(needle) + p.tag("fun_post", p.negate(p.space() + p.literal("[") + p.until("]") + p.literal("]")) + p.space()) })); }); auto result = parser.parse_anywhere_and_extract(haystack); t.assert_true("success", result.result.success()); t.assert_equal("fun_pre should be ''", ">", result.tags["fun_post"]); }); } static void test_permute(testing & t) { auto accepts = [](const common_peg_arena & parser, const std::string & input) { common_peg_parse_context ctx(input); return parser.parse(ctx).success(); }; auto gbnf_of = [](const common_peg_arena & parser) { return build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); }; auto assert_gbnf_equal = [](testing & t, const std::string & expected, const std::string & actual) { static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)"); t.assert_equal("gbnf are equal", std::regex_replace(expected, leading_ws_re, "$1"), actual); }; auto count_rules = [](const std::string & gbnf, const std::string & prefix) { size_t count = 0; for (const auto & line : string_split(gbnf, '\n')) { if (line.rfind(prefix, 0) == 0) { count++; } } return count; }; t.test("accepts every ordering", [&](testing & t) { auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end(); }); for (const std::string input : { "abc", "acb", "bac", "bca", "cab", "cba" }) { t.assert_true("accepts " + input, accepts(parser, input)); } }); t.test("single element", [&](testing & t) { auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { return p.permute("a", { p.literal("a") }) + p.end(); }); t.assert_true("accepts a", accepts(parser, "a")); t.assert_true("rejects aa", !accepts(parser, "aa")); }); t.test("grammar left-factorizes shared tails", [&](testing & t) { auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end(); }); // Every rule is one remaining subset, keyed by bitmask: abc-3 is {a,b}, abc-7 is {a,b,c}. // Each subset is emitted once and shared by every branch that leads into it. assert_gbnf_equal(t, R"""( abc-1 ::= "a" abc-2 ::= "b" abc-3 ::= "a" abc-2 | "b" abc-1 abc-4 ::= "c" abc-5 ::= "a" abc-4 | "c" abc-1 abc-6 ::= "b" abc-4 | "c" abc-2 abc-7 ::= "a" abc-6 | "b" abc-5 | "c" abc-3 root ::= abc-7 space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf_of(parser)); }); t.test("grammar emits one rule per remaining subset", [&](testing & t) { auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { return p.permute("abcd", { p.literal("a"), p.literal("b"), p.literal("c"), p.literal("d") }) + p.end(); }); // 2^4 - 1 non-empty subsets, one rule each - not the 4! = 24 orderings. t.assert_equal("permute rule count", 15u, count_rules(gbnf_of(parser), "abcd-")); }); t.test("grammar emits no rules for a single element", [&](testing & t) { auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { return p.permute("a", { p.literal("a") }) + p.end(); }); assert_gbnf_equal(t, R"""( root ::= "a" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf_of(parser)); }); t.test("grammar falls back to the given order when too large", [&](testing & t) { auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { std::vector parsers; for (size_t i = 0; i <= COMMON_CHAT_MAX_PERMUTE; i++) { parsers.push_back(p.literal(std::string(1, (char) ('a' + i)))); } return p.permute("big", parsers) + p.end(); }); assert_gbnf_equal(t, R"""( root ::= "a" "b" "c" "d" "e" "f" "g" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf_of(parser)); }); }