Something went wrong. Try again.
llama.cpp fork for Ternary Bonsai 2 27B: MTP speculation, suffix cache reuse (SCR), q8 KV, two-slot fairness, fused DeltaNet kernels
Something went wrong. Try again.
863 B · 37 lines
C++
at main
1234567891011121314151617181920212223242526272829303132333435363738#include "simple-tokenize.h"
std::vector<std::string> simple_tokenize(const std::string & input) { std::vector<std::string> result; std::string current;
for (size_t i = 0; i < input.size(); i++) { switch (input[i]) { case ' ': case '\n': case '\t': case '{': case '}': case ',': case '[': case '"': case ']': case '.': case '<': case '>': case '=': case '/': if (!current.empty()) { result.push_back(current); current.clear(); } default:; } current += input[i]; }
if (!current.empty()) { result.push_back(current); }
return result;}