Something went wrong. Try again.
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228diff --git a/common/arg.cpp b/common/arg.cppindex e096bfad..be415ee0 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -1724,6 +1724,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.cache_disk_mib = value; } ).set_env("LLAMA_ARG_CACHE_DISK_SIZE").set_examples({LLAMA_EXAMPLE_SERVER}));+ add_opt(common_arg(+ {"--cache-embd-ram"}, "N",+ string_format("RAM for image embeddings by image content in MiB, so an image that has to be prefilled again\n"+ "skips the vision encoder (default: %d, 0 - off)", params.cache_embd_ram_mib),+ [](common_params & params, int value) {+ params.cache_embd_ram_mib = value;+ }+ ).set_env("LLAMA_ARG_CACHE_EMBD_RAM").set_examples({LLAMA_EXAMPLE_SERVER}));+ add_opt(common_arg(+ {"--cache-embd-disk"}, "N",+ string_format("disk space for image embeddings under <--cache-disk>/embd in MiB (default: %d, 0 - off)", params.cache_embd_disk_mib),+ [](common_params & params, int value) {+ params.cache_embd_disk_mib = value;+ }+ ).set_env("LLAMA_ARG_CACHE_EMBD_DISK").set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"--cache-ttl-ephemeral"}, "N", string_format("seconds a prompt cache entry of class ephemeral lives after its last use (default: %d)", params.cache_ttl_ephemeral),diff --git a/common/common.h b/common/common.hindex 550bef61..cc89d4b1 100644--- a/common/common.h+++ b/common/common.h@@ -621,6 +621,8 @@ struct common_params { int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string cache_disk_path; // directory of the prompt cache's disk tier, empty = off int32_t cache_disk_mib = 51200; // disk tier size limit in MiB, 0 = no limit+ int32_t cache_embd_ram_mib = 256; // image embedding cache in RAM, MiB, 0 = off+ int32_t cache_embd_disk_mib = 4096; // image embedding cache under <cache_disk_path>/embd, MiB, 0 = off int32_t cache_ttl_ephemeral = 3600; // seconds an unused prompt cache entry of each class is kept int32_t cache_ttl_default = 3*86400; int32_t cache_ttl_keep = 30*86400;diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txtindex 280bd9e1..1f6ad1f2 100644--- a/tools/server/CMakeLists.txt+++ b/tools/server/CMakeLists.txt@@ -15,6 +15,8 @@ add_library(${TARGET} STATIC server-common.h server-context.cpp server-context.h+ server-embd-cache.cpp+ server-embd-cache.h server-stream.cpp server-stream.h server-tools.cppdiff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex f657fdf0..0609fa71 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -1,6 +1,7 @@ #include "server-context.h" #include "server-chat.h" #include "server-common.h"+#include "server-embd-cache.h" #include "server-http.h" #include "server-task.h" #include "server-queue.h"@@ -205,6 +206,7 @@ struct server_slot { // multimodal mtmd_context * mctx = nullptr; mtmd::batch_ptr mbatch = nullptr;+ server_embd_cache * embd_cache = nullptr; // may be null // speculative decoding common_speculative * spec;@@ -734,38 +736,51 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch const auto & chunk = input_tokens.find_chunk(idx); int32_t res = 0; + // an image's rows are cached by its content id and token grid (the decoder position of its last token)+ const mtmd_image_tokens * image = mtmd_input_chunk_get_tokens_image(chunk.get());+ server_embd_cache * cache = image ? slot.embd_cache : nullptr;+ const mtmd_decoder_pos grid = image ? mtmd_image_tokens_get_decoder_pos(image, 0, mtmd_image_tokens_get_n_tokens(image) - 1) : mtmd_decoder_pos{};+ const size_t n_floats = (size_t) llama_model_n_embd_inp(llama_get_model(slot.ctx_tgt)) * mtmd_input_chunk_get_n_tokens(chunk.get());++ auto decode = [&](float * embd) -> int32_t {+ void * cb_data = slot.spec;+ static auto cb = [](llama_batch batch, void * user_data) {+ common_speculative * spec = static_cast<common_speculative *>(user_data);+ if (!common_speculative_process(spec, batch)) {+ return 1;+ }+ return 0;+ };++ llama_pos new_n_past; // unused for now+ res = mtmd_helper_decode_image_chunk(+ mctx,+ slot.ctx_tgt,+ chunk.get(),+ embd,+ slot.prompt.tokens.pos_next(),+ slot.id,+ llama_n_batch(slot.ctx_tgt),+ &new_n_past,+ cb,+ cb_data+ );+ if (res != 0) {+ SLT_ERR(slot, "failed to decode mtmd chunk, idx = %zu, res = %d\n", idx, res);+ return -1;+ }+ n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get());+ return 0; // success+ };+ auto try_decode = [&]() -> int32_t { if (mbatch) { float * embd = mtmd_batch_get_output_embd(mbatch.get(), chunk.get()); if (embd) {- void * cb_data = slot.spec;- static auto cb = [](llama_batch batch, void * user_data) {- common_speculative * spec = static_cast<common_speculative *>(user_data);- if (!common_speculative_process(spec, batch)) {- return 1;- }- return 0;- };-- llama_pos new_n_past; // unused for now- res = mtmd_helper_decode_image_chunk(- mctx,- slot.ctx_tgt,- chunk.get(),- embd,- slot.prompt.tokens.pos_next(),- slot.id,- llama_n_batch(slot.ctx_tgt),- &new_n_past,- cb,- cb_data- );- if (res != 0) {- SLT_ERR(slot, "failed to decode mtmd chunk, idx = %zu, res = %d\n", idx, res);- return -1;+ if (cache) {+ cache->put(mtmd_input_chunk_get_id(chunk.get()), grid.x + 1, grid.y + 1, embd, n_floats); }- n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get());- return 0; // success+ return decode(embd); } } return 1; // (non-error) need to create & encode batch@@ -781,6 +796,16 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch return res; } + if (cache) {+ const int64_t t0 = ggml_time_us();+ std::vector<float> rows;+ if (cache->get(mtmd_input_chunk_get_id(chunk.get()), grid.x + 1, grid.y + 1, n_floats, rows)) {+ SLT_INF(slot, "image at idx = %zu (%zu tokens): cached embedding in %.1f ms, encoder skipped\n", idx,+ mtmd_input_chunk_get_n_tokens(chunk.get()), (ggml_time_us() - t0) / 1e3);+ return decode(rows.data());+ }+ }+ // otherwise, the batch is either uninitialized or is used up // we need to create & encode a new batch mbatch.reset(mtmd_batch_init(mctx));@@ -805,11 +830,13 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch // TODO @ngxson : move this log line to debug when it become more stable SLT_TRC(slot, "encoding mtmd batch from idx = %zu, n_chunks = %d\n", idx, n_added); + const int64_t t_encode = ggml_time_us(); res = mtmd_batch_encode(mbatch.get()); if (res != 0) { SLT_ERR(slot, "failed to encode mtmd batch for chunk idx = %zu, res = %d\n", idx, res); return -1; }+ SLT_INF(slot, "encoded %d media chunk(s) from idx = %zu in %.1f ms\n", n_added, idx, (ggml_time_us() - t_encode) / 1e3); return try_decode(); }@@ -900,6 +927,20 @@ private: std::unique_ptr<server_prompt_cache> prompt_cache; + // created with the multimodal projector, kept across sleep+ std::unique_ptr<server_embd_cache> embd_cache;++ // what the image embeddings depend on besides the image: the projector file and the encoder settings+ std::string media_identity() const {+ std::error_code ec;+ const auto & path = params_base.mmproj.path;+ const auto mtime = std::filesystem::last_write_time(path, ec).time_since_epoch().count();+ const auto size = std::filesystem::file_size(path, ec);+ const char * bi = getenv("GGML_CUDA_BATCH_INVARIANT");+ return string_format("%s|%llu|%lld|gpu=%d|min=%d|max=%d|bi=%s", path.c_str(), (unsigned long long) size, (long long) mtime,+ params_base.mmproj_use_gpu ? 1 : 0, params_base.image_min_tokens, params_base.image_max_tokens, bi ? bi : "");+ }+ server_metrics metrics; // queued prompt stats - llama_decode() is async, so the timing is only valid after a sync@@ -1169,6 +1210,14 @@ private: } SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str()); + if (!embd_cache && (params_base.cache_embd_ram_mib > 0 || params_base.cache_embd_disk_mib > 0)) {+ const bool disk = !params_base.cache_disk_path.empty() && params_base.cache_embd_disk_mib > 0;+ embd_cache = std::make_unique<server_embd_cache>(media_identity(),+ 1024ull*1024ull*(size_t) std::max(params_base.cache_embd_ram_mib, 0),+ disk ? params_base.cache_disk_path + "/embd" : "",+ 1024ull*1024ull*(size_t) std::max(params_base.cache_embd_disk_mib, 0));+ }+ if (params_base.ctx_shift) { params_base.ctx_shift = false; SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");@@ -1292,6 +1341,7 @@ private: slot.stats.speculative = slot.can_speculate(); slot.mctx = mctx;+ slot.embd_cache = embd_cache.get(); slot.prompt.tokens.has_mtmd = mctx != nullptr; SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);@@ -1393,7 +1443,7 @@ private: (unsigned long long) llama_model_size(model_tgt), (unsigned long long) llama_model_n_params(model_tgt), (long long) mtime, ggml_type_name(params_base.cache_type_k), ggml_type_name(params_base.cache_type_v), ggml_type_name(params_base.speculative.draft.cache_type_k), ggml_type_name(params_base.speculative.draft.cache_type_v));- prompt_cache->disk_init(params_base.cache_disk_path, params_base.cache_disk_mib, key);+ prompt_cache->disk_init(params_base.cache_disk_path, params_base.cache_disk_mib, key, mctx ? media_identity() : ""); } prompt_cache->policy_start(); } else {@@ -2468,10 +2518,19 @@ private: return scr_pays(plan, task.tokens.size()) ? plan.n_prefix + plan.n_reused() : (int32_t) prompt.tokens.get_common_prefix(task.tokens); }; + // loading an entry consumes it, so one counts only for a splice that pays or, like in prompt_cache.load(),+ // for a plain prefix of at least a quarter of it: a few tokens of chat template must not take an entry. // planning against every cached prompt is O(cache size); only prompts that start like this one are planned const auto value_of = [&](const server_prompt & prompt, int32_t n_state) { const int32_t lcp = prompt.tokens.get_common_prefix(task.tokens);- return lcp < 16 ? lcp : value(scr_plan(prompt, n_state, task.tokens), prompt);+ if (lcp < 16) {+ return -1;+ }+ const common_scr_plan plan = scr_plan(prompt, n_state, task.tokens);+ if (scr_pays(plan, task.tokens.size())) {+ return plan.n_prefix + plan.n_reused();+ }+ return 4*lcp >= prompt.n_tokens() ? lcp : -1; }; const int32_t n_own = value(scr_plan(slot, task.tokens), slot.prompt);@@ -2490,10 +2549,10 @@ private: } } - // disk entries are text prompts without their checkpoints in the index- const std::string path = prompt_cache->disk_best([&](const llama_tokens & tokens) {+ // the disk index holds the prompts (images as placeholder chunks) without their checkpoints+ const std::string path = prompt_cache->disk_best([&](const server_tokens & tokens) { server_prompt prompt;- prompt.tokens = server_tokens(tokens, false);+ prompt.tokens = tokens.clone(); return value_of(prompt, prompt.n_tokens()); }, n_best); @@ -2506,7 +2565,7 @@ private: } // saving can evict RAM entries, so a RAM entry is looked up again by its id- auto best = !path.empty() ? prompt_cache->disk_promote(path, task.tokens.has_mtmd) :+ auto best = !path.empty() ? prompt_cache->disk_promote(path) : std::find_if(prompt_cache->states.begin(), prompt_cache->states.end(), [&](const server_prompt_cache_state & s) { return s.prompt.meta.id == id_best; }); if (best == prompt_cache->states.end()) { return;@@ -3077,8 +3136,8 @@ private: if (task.type == SERVER_TASK_TYPE_CACHE_CLEAR) { const std::string & scope = task.cache_scope; const bool all = scope == "all";- if (!all && scope != "slots" && scope != "ram" && scope != "disk") {- send_error(task, "scope must be slots, ram, disk or all", ERROR_TYPE_INVALID_REQUEST);+ if (!all && scope != "slots" && scope != "ram" && scope != "disk" && scope != "embd") {+ send_error(task, "scope must be slots, ram, disk, embd or all", ERROR_TYPE_INVALID_REQUEST); break; } if (all || scope == "slots") {@@ -3103,6 +3162,9 @@ private: if (prompt_cache && (all || scope == "disk")) { cleared["disk"] = prompt_cache->clear_disk(task.cache_include_pinned); }+ if (embd_cache && (all || scope == "embd")) {+ cleared["embd"] = embd_cache->clear();+ } SRV_INF("cache cleared (%s): %s\n", scope.c_str(), cleared.dump().c_str()); } @@ -3122,6 +3184,9 @@ private: res->id = task.id; res->data = prompt_cache ? prompt_cache->info() : json::object(); res->data["slots"] = slots_info;+ if (embd_cache) {+ res->data["embd"] = embd_cache->info();+ } if (task.type == SERVER_TASK_TYPE_CACHE_CLEAR) { res->data["cleared"] = cleared; }@@ -5243,7 +5308,7 @@ void server_routes::init_routes() { return res; }; - // GET /cache: slots and both prompt-cache tiers; POST /cache/clear?scope=slots|ram|disk|all+ // GET /cache: slots, both prompt-cache tiers and the image embedding cache; POST /cache/clear?scope=slots|ram|disk|embd|all this->get_cache = [this](const server_http_req & req) { return handle_cache(req, SERVER_TASK_TYPE_CACHE_GET, ""); };diff --git a/tools/server/server-embd-cache.cpp b/tools/server/server-embd-cache.cppnew file mode 100644index 00000000..5d4035ae--- /dev/null+++ b/tools/server/server-embd-cache.cpp@@ -0,0 +1,275 @@+#include "server-embd-cache.h"++#include <algorithm>+#include <cstdio>+#include <cstring>+#include <filesystem>++namespace {++constexpr uint32_t EMBD_MAGIC = 0x424d4542; // "BEMB"+constexpr uint32_t EMBD_VERSION = 1;++// magic, version, identity, nx, ny, n_floats, then the rows and a checksum of their bytes+constexpr size_t EMBD_HEADER = 4 + 4 + 8 + 4 + 4 + 8;++uint64_t fnv1a(const void * data, size_t n, uint64_t h = 1469598103934665603ull) {+ const auto * p = static_cast<const uint8_t *>(data);+ for (size_t i = 0; i < n; ++i) {+ h = (h ^ p[i]) * 1099511628211ull;+ }+ return h;+}++// fnv-1a over 64-bit words: a corruption check, ~8x faster than bytewise on 20 MiB of rows+uint64_t checksum(const float * data, size_t n_floats) {+ const size_t n_bytes = n_floats * sizeof(float);+ uint64_t h = 1469598103934665603ull;+ size_t i = 0;+ for (; i + 8 <= n_bytes; i += 8) {+ uint64_t w;+ memcpy(&w, (const uint8_t *) data + i, 8);+ h = (h ^ w) * 1099511628211ull;+ }+ return fnv1a((const uint8_t *) data + i, n_bytes - i, h);+}++std::string ram_key(const std::string & id, uint32_t nx, uint32_t ny) {+ return id + "-" + std::to_string(nx) + "x" + std::to_string(ny);+}++// ids are content hashes (hex); anything else stays out of file names+bool valid_id(const std::string & id) {+ return !id.empty() && id.size() <= 128 &&+ std::all_of(id.begin(), id.end(), [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); });+}++struct embd_file {+ FILE * f;+ embd_file(const std::string & path, const char * mode) : f(fopen(path.c_str(), mode)) {}+ ~embd_file() {+ if (f) {+ fclose(f);+ }+ }+};++template <typename T> bool write_val(FILE * f, const T & v) {+ return fwrite(&v, sizeof(v), 1, f) == 1;+}++template <typename T> bool read_val(FILE * f, T & v) {+ return fread(&v, sizeof(v), 1, f) == 1;+}++bool read_header(FILE * f, uint64_t ident, uint32_t & nx, uint32_t & ny, uint64_t & n_floats) {+ uint32_t magic, version;+ uint64_t id;+ return read_val(f, magic) && read_val(f, version) && read_val(f, id) && read_val(f, nx) && read_val(f, ny) && read_val(f, n_floats) &&+ magic == EMBD_MAGIC && version == EMBD_VERSION && id == ident && n_floats < (1ull << 32);+}++} // namespace++server_embd_cache::server_embd_cache(const std::string & identity, size_t ram_limit, const std::string & dir, size_t disk_limit)+ : ident(fnv1a(identity.data(), identity.size())), ram_limit(ram_limit), dir(dir), disk_limit(disk_limit) {+ namespace fs = std::filesystem;++ if (this->dir.empty()) {+ return;+ }+ std::error_code ec;+ fs::create_directories(this->dir, ec);+ if (ec) {+ SRV_ERR("embedding cache: cannot create %s: %s, disk tier off\n", this->dir.c_str(), ec.message().c_str());+ this->dir.clear();+ return;+ }++ // oldest first, so the clock replays the files' order of use+ std::vector<std::pair<fs::file_time_type, std::string>> found;+ for (const auto & de : fs::directory_iterator(this->dir, ec)) {+ const auto ext = de.path().extension();+ if (ext == ".tmp") {+ fs::remove(de.path(), ec);+ continue;+ }+ if (ext != ".emb") {+ continue;+ }+ uint32_t nx, ny;+ uint64_t n_floats;+ bool ok;+ {+ embd_file f(de.path().string(), "rb");+ ok = f.f && read_header(f.f, ident, nx, ny, n_floats) &&+ de.file_size(ec) == EMBD_HEADER + n_floats * sizeof(float) + sizeof(uint64_t);+ }+ if (!ok) {+ // another mmproj or encoder setting+ fs::remove(de.path(), ec);+ continue;+ }+ found.push_back({ de.last_write_time(ec), de.path().filename().string() });+ }+ std::sort(found.begin(), found.end());+ for (const auto & [_, name] : found) {+ const size_t size = fs::file_size(this->dir + "/" + name, ec);+ disk[name] = { size, ++disk_clock };+ disk_bytes += size;+ }+ disk_evict_locked();++ SRV_INF("embedding cache: RAM %.0f MiB, disk %s with %zu images, %.1f MiB (limit %.0f MiB)\n", ram_limit / 1048576.0,+ this->dir.c_str(), disk.size(), disk_bytes / 1048576.0, disk_limit / 1048576.0);+}++bool server_embd_cache::get(const std::string & id, uint32_t nx, uint32_t ny, size_t n_floats, std::vector<float> & out) {+ std::lock_guard<std::mutex> lock(mutex);++ const std::string key = ram_key(id, nx, ny);+ const auto it = ram_index.find(key);+ if (it != ram_index.end() && it->second->data.size() == n_floats) {+ ram.splice(ram.end(), ram, it->second);+ out = it->second->data;+ n_hits_ram++;+ return true;+ }++ if (!dir.empty() && valid_id(id) && disk_read_locked(key + ".emb", n_floats, out)) {+ ram_put_locked(key, std::vector<float>(out));+ n_hits_disk++;+ return true;+ }++ n_misses++;+ return false;+}++void server_embd_cache::put(const std::string & id, uint32_t nx, uint32_t ny, const float * data, size_t n_floats) {+ std::lock_guard<std::mutex> lock(mutex);++ const std::string key = ram_key(id, nx, ny);+ if (ram_index.count(key) == 0) {+ ram_put_locked(key, std::vector<float>(data, data + n_floats));+ }+ if (!dir.empty() && valid_id(id) && disk.count(key + ".emb") == 0) {+ disk_write_locked(key + ".emb", nx, ny, data, n_floats);+ }+}++void server_embd_cache::ram_put_locked(const std::string & key, std::vector<float> && data) {+ const size_t size = data.size() * sizeof(float);+ if (size > ram_limit) {+ return;+ }+ while (!ram.empty() && ram_bytes + size > ram_limit) {+ ram_bytes -= ram.front().data.size() * sizeof(float);+ ram_index.erase(ram.front().key);+ ram.pop_front();+ }+ ram.push_back({ key, std::move(data) });+ ram_index[key] = std::prev(ram.end());+ ram_bytes += size;+}++bool server_embd_cache::disk_read_locked(const std::string & name, size_t n_floats, std::vector<float> & out) {+ const auto it = disk.find(name);+ if (it == disk.end()) {+ return false;+ }+ const std::string path = dir + "/" + name;+ bool ok;+ {+ embd_file f(path, "rb");+ uint32_t nx, ny;+ uint64_t n, sum;+ ok = f.f && read_header(f.f, ident, nx, ny, n) && n == n_floats;+ if (ok) {+ out.resize(n_floats);+ ok = fread(out.data(), sizeof(float), n_floats, f.f) == n_floats && read_val(f.f, sum) && sum == checksum(out.data(), n_floats);+ }+ }+ if (!ok) {+ SRV_WRN("embedding cache: cannot read %s, removing it\n", path.c_str());+ std::error_code ec;+ std::filesystem::remove(path, ec);+ disk_bytes -= it->second.size;+ disk.erase(it);+ return false;+ }+ it->second.used = ++disk_clock;+ std::error_code ec;+ std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now(), ec);+ return true;+}++void server_embd_cache::disk_write_locked(const std::string & name, uint32_t nx, uint32_t ny, const float * data, size_t n_floats) {+ const size_t size = EMBD_HEADER + n_floats * sizeof(float) + sizeof(uint64_t);+ if (size > disk_limit) {+ return;+ }+ const std::string path = dir + "/" + name;+ const std::string tmp = path + ".tmp";++ bool ok;+ {+ embd_file f(tmp, "wb");+ ok = f.f && write_val(f.f, EMBD_MAGIC) && write_val(f.f, EMBD_VERSION) && write_val(f.f, ident) && write_val(f.f, nx) && write_val(f.f, ny) &&+ write_val(f.f, (uint64_t) n_floats) && fwrite(data, sizeof(float), n_floats, f.f) == n_floats &&+ write_val(f.f, checksum(data, n_floats)) && fflush(f.f) == 0;+ }+ std::error_code ec;+ if (ok) {+ std::filesystem::rename(tmp, path, ec);+ ok = !ec;+ }+ if (!ok) {+ SRV_WRN("embedding cache: cannot write %s\n", path.c_str());+ std::filesystem::remove(tmp, ec);+ return;+ }+ disk[name] = { size, ++disk_clock };+ disk_bytes += size;+ disk_evict_locked();+}++void server_embd_cache::disk_evict_locked() {+ std::error_code ec;+ while (disk_bytes > disk_limit && !disk.empty()) {+ auto victim = std::min_element(disk.begin(), disk.end(), [](const auto & a, const auto & b) { return a.second.used < b.second.used; });+ std::filesystem::remove(dir + "/" + victim->first, ec);+ disk_bytes -= victim->second.size;+ disk.erase(victim);+ }+}++json server_embd_cache::info() const {+ std::lock_guard<std::mutex> lock(mutex);+ json res = {+ { "ram", { { "images", ram.size() }, { "bytes", ram_bytes }, { "limit_bytes", ram_limit } } },+ { "hits_ram", n_hits_ram },+ { "hits_disk", n_hits_disk },+ { "misses", n_misses },+ };+ if (!dir.empty()) {+ res["disk"] = { { "dir", dir }, { "images", disk.size() }, { "bytes", disk_bytes }, { "limit_bytes", disk_limit } };+ }+ return res;+}++size_t server_embd_cache::clear() {+ std::lock_guard<std::mutex> lock(mutex);+ size_t n = ram.size();+ ram.clear();+ ram_index.clear();+ ram_bytes = 0;+ std::error_code ec;+ for (const auto & [name, _] : disk) {+ std::filesystem::remove(dir + "/" + name, ec);+ }+ n += disk.size();+ disk.clear();+ disk_bytes = 0;+ return n;+}diff --git a/tools/server/server-embd-cache.h b/tools/server/server-embd-cache.hnew file mode 100644index 00000000..f5ce9488--- /dev/null+++ b/tools/server/server-embd-cache.h@@ -0,0 +1,64 @@+#pragma once++#include "server-common.h"++#include <cstdint>+#include <list>+#include <mutex>+#include <string>+#include <unordered_map>+#include <vector>++// image embeddings (the vision encoder's output rows) by image content, so an image the prompt cache cannot+// reuse is decoded without running the encoder again. a RAM LRU tier and an optional disk tier, each under a+// byte limit. the key folds in everything else the rows depend on (mmproj file, image token limits, encoder+// device), so a cached row is the one a fresh encode of the same image computes. thread-safe+struct server_embd_cache {+ // identity: mmproj file and encoder settings; dir empty = no disk tier+ server_embd_cache(const std::string & identity, size_t ram_limit, const std::string & dir, size_t disk_limit);++ // the rows of the image with this content id and token grid; false if not cached+ bool get(const std::string & id, uint32_t nx, uint32_t ny, size_t n_floats, std::vector<float> & out);++ void put(const std::string & id, uint32_t nx, uint32_t ny, const float * data, size_t n_floats);++ json info() const;++ // drops both tiers; returns the number of entries dropped+ size_t clear();++private:+ struct ram_entry {+ std::string key;+ std::vector<float> data;+ };++ struct disk_entry {+ size_t size;+ int64_t used; // order of use, larger = more recent+ };++ uint64_t ident;+ size_t ram_limit;+ std::string dir;+ size_t disk_limit;++ mutable std::mutex mutex;++ std::list<ram_entry> ram; // most recently used last+ std::unordered_map<std::string, std::list<ram_entry>::iterator> ram_index;+ size_t ram_bytes = 0;++ std::unordered_map<std::string, disk_entry> disk; // by file name+ size_t disk_bytes = 0;+ int64_t disk_clock = 0;++ uint64_t n_hits_ram = 0;+ uint64_t n_hits_disk = 0;+ uint64_t n_misses = 0;++ void ram_put_locked(const std::string & key, std::vector<float> && data);+ bool disk_read_locked(const std::string & name, size_t n_floats, std::vector<float> & out);+ void disk_write_locked(const std::string & name, uint32_t nx, uint32_t ny, const float * data, size_t n_floats);+ void disk_evict_locked();+};diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex 0a83f4d0..044e2480 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -1743,8 +1743,12 @@ namespace { constexpr uint32_t DISK_MAGIC = 0x43564b42; // "BKVC" constexpr uint32_t DISK_VERSION = 2;+constexpr uint32_t DISK_VERSION_MEDIA = 3; -// v2: magic, version, key, META_SIZE bytes of metadata (rewritten in place), then the v1 body+// v2: magic, version, key, META_SIZE bytes of metadata (rewritten in place), then the v1 body.+// v3 (prompts with images): as v2, but the token list is the media key followed by server_tokens::serialize()+// (tokens plus the placeholder media chunks: id, grid, position layout). text prompts stay v2, so a binary that+// predates v3 deletes only the image files (it reads them as another format), and keeps the rest constexpr size_t META_OFFSET = 16; constexpr size_t META_SIZE = 512; constexpr size_t META_FIXED = 74;@@ -1993,29 +1997,83 @@ bool get_bytes(FILE * f, std::vector<uint8_t> & v) { return get(f, v.data(), n); } -// magic, version, key, metadata (v2) and the prompt tokens: enough to index a file without reading its state.-// a v1 file has no metadata; has_meta is false for it-bool read_header(FILE * f, uint64_t key, llama_tokens & tokens, uint32_t & version, cache_meta & meta) {+// the number of media chunks in t, or -1 if its chunk map does not cover exactly its media positions+int64_t media_count(const server_tokens & t) {+ int64_t n = 0;+ for (size_t i = 0; i < t.size();) {+ if (t[i] != LLAMA_TOKEN_NULL) {+ ++i;+ continue;+ }+ const auto [s, e] = t.media_span(i);+ if (s != i || e <= i || e > t.size()) {+ return -1;+ }+ for (size_t j = s; j < e; ++j) {+ if (t[j] != LLAMA_TOKEN_NULL) {+ return -1;+ }+ }+ n++;+ i = e;+ }+ // chunks at text positions or past the end+ int64_t n_map = 0;+ try {+ t.find_chunk(0);+ n_map++;+ } catch (const std::exception &) {+ }+ for (auto p = t.find_next_media_chunk(0); p.first; p = t.find_next_media_chunk(p.second)) {+ n_map++;+ }+ return n_map == n ? n : -1;+}++// magic, version, key, metadata (v2, v3) and the prompt tokens with their media chunks (v3): enough to index a+// file without reading its state. a v1 file has no metadata. a v3 file is only valid under the same media key+bool read_header(FILE * f, uint64_t key, uint64_t media, server_tokens & tokens, uint32_t & version, cache_meta & meta) { uint32_t magic; uint64_t k; uint64_t n; if (!get(f, magic) || !get(f, version) || !get(f, k)) { return false; }- if (magic != DISK_MAGIC || (version != 1 && version != DISK_VERSION) || k != key) {+ if (magic != DISK_MAGIC || (version != 1 && version != DISK_VERSION && version != DISK_VERSION_MEDIA) || k != key) { return false; }- if (version == DISK_VERSION) {+ if (version >= DISK_VERSION) { std::vector<uint8_t> b(META_SIZE); if (!get(f, b.data(), META_SIZE) || !meta_decode(b, meta)) { return false; } }+ if (version == DISK_VERSION_MEDIA) {+ uint64_t m;+ if (!get(f, m) || m != media || media == 0) {+ return false;+ }+ } if (!get(f, n) || n > (1ull << 26)) { return false; }- tokens.resize(n);- return get(f, tokens.data(), n * sizeof(llama_token));+ llama_tokens raw(n);+ if (!get(f, raw.data(), n * sizeof(llama_token))) {+ return false;+ }+ if (version != DISK_VERSION_MEDIA) {+ tokens = server_tokens(raw, media != 0);+ return true;+ }+ if (raw.empty() || raw[0] != LLAMA_TOKEN_NULL) {+ return false;+ }+ try {+ tokens = server_tokens::deserialize(raw, true);+ } catch (const std::exception &) {+ return false;+ }+ return media_count(tokens) > 0; } bool write_meta(const std::string & path, const cache_meta & meta) {@@ -2027,15 +2085,6 @@ bool write_meta(const std::string & path, const cache_meta & meta) { return put(f.f, b.data(), b.size()) && fflush(f.f) == 0; } -size_t common_prefix(const llama_tokens & a, const server_tokens & b) {- const size_t n = std::min(a.size(), b.size());- size_t i = 0;- while (i < n && b[i] != LLAMA_TOKEN_NULL && a[i] == b[i]) {- ++i;- }- return i;-}- // the token list, media positions as LLAMA_TOKEN_NULL (get_tokens() refuses a multimodal server) llama_tokens raw_tokens(const server_tokens & t) { llama_tokens res(t.size());@@ -2045,9 +2094,18 @@ llama_tokens raw_tokens(const server_tokens & t) { return res; } -// a is a prefix of b-bool is_prefix(const llama_tokens & a, const llama_tokens & b) {- return a.size() <= b.size() && std::equal(a.begin(), a.end(), b.begin());+bool has_media(const server_tokens & t) {+ for (size_t i = 0; i < t.size(); ++i) {+ if (t[i] == LLAMA_TOKEN_NULL) {+ return true;+ }+ }+ return false;+}++// a is a prefix of b; a media chunk matches by id and size+bool is_prefix(const server_tokens & a, const server_tokens & b) {+ return a.size() <= b.size() && a.get_common_prefix(b) == a.size(); } size_t count_seq(const llama_tokens & tokens, const llama_tokens & seq) {@@ -2063,13 +2121,14 @@ size_t count_seq(const llama_tokens & tokens, const llama_tokens & seq) { return n; } -json entry_json(const cache_meta & m, size_t n_tokens, size_t bytes, int64_t now) {+json entry_json(const cache_meta & m, size_t n_tokens, size_t n_images, size_t bytes, int64_t now) { return { { "id", hex_id(m.id) }, { "class", cache_class_name(m.cls) }, { "source", cache_source_name(m.source) }, { "classified", m.classified() }, { "n_tokens", n_tokens },+ { "images", n_images }, { "bytes", bytes }, { "hits", m.hits }, { "age", now - m.created },@@ -2089,7 +2148,7 @@ struct tier_report { std::map<std::string, std::pair<size_t, size_t>> by_class; std::vector<std::pair<int64_t, json>> entries; - void add(const cache_meta & m, size_t n_tokens, size_t n_bytes, int64_t now) {+ void add(const cache_meta & m, size_t n_tokens, size_t n_images, size_t n_bytes, int64_t now) { prompts++; tokens += n_tokens; bytes += n_bytes;@@ -2099,7 +2158,7 @@ struct tier_report { auto & c = by_class[cache_class_name(m.cls)]; c.first++; c.second += n_bytes;- entries.push_back({ m.last_used, entry_json(m, n_tokens, n_bytes, now) });+ entries.push_back({ m.last_used, entry_json(m, n_tokens, n_images, n_bytes, now) }); } void to_json(json & out) {@@ -2288,7 +2347,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok expire_disk_locked(); for (const auto & e : disk) {- const int lcp_cur = common_prefix(e.tokens, tokens_new);+ const int lcp_cur = e.tokens.get_common_prefix(tokens_new); const float f_keep_cur = float(lcp_cur) / e.tokens.size(); const float f_sim_cur = float(lcp_cur) / tokens_new.size();@@ -2309,7 +2368,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok bool from_disk = false; if (!disk_best.empty()) {- const auto it = disk_promote(disk_best, tokens_new.has_mtmd);+ const auto it = disk_promote(disk_best); if (it != states.end()) { SRV_INF("prompt cache: f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); it_best = it;@@ -2331,12 +2390,12 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok return true; } -std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote(const std::string & path, bool has_mtmd) {+std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote(const std::string & path) { namespace fs = std::filesystem; const int64_t t_start = ggml_time_us(); server_prompt_cache_state state;- const bool ok = disk_read(path, has_mtmd, state);+ const bool ok = disk_read(path, state); std::lock_guard<std::mutex> lock(disk_mutex); std::error_code ec;@@ -2350,7 +2409,8 @@ std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote return states.end(); } - SRV_INF("prompt cache: read %zu tokens from disk in %.2f s\n", state.prompt.tokens.size(), (ggml_time_us() - t_start) / 1e6);+ SRV_INF("prompt cache: read %zu tokens, %lld images from disk in %.2f s\n", state.prompt.tokens.size(),+ (long long) std::max<int64_t>(0, media_count(state.prompt.tokens)), (ggml_time_us() - t_start) / 1e6); if (it != disk.end()) { // the file stays: it records the use, and is most recently used, here and after a restart it->meta.last_used = unix_now();@@ -2358,7 +2418,7 @@ std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote set_expiry(it->meta); state.prompt.meta = it->meta; state.prompt.meta.hits--; // the slot counts this use when the request continues the prompt- if (it->version == DISK_VERSION) {+ if (it->version >= DISK_VERSION) { write_meta(it->path, it->meta); } disk.splice(disk.end(), disk, it);@@ -2369,7 +2429,7 @@ std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote return std::prev(states.end()); } -std::string server_prompt_cache::disk_best(const std::function<int32_t(const llama_tokens &)> & score, int32_t & n_best) {+std::string server_prompt_cache::disk_best(const std::function<int32_t(const server_tokens &)> & score, int32_t & n_best) { std::string best; std::lock_guard<std::mutex> lock(disk_mutex); if (!disk_thread.joinable()) {@@ -3002,12 +3062,13 @@ server_prompt_cache::~server_prompt_cache() { } } -void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, const std::string & key) {+void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, const std::string & key, const std::string & media) { namespace fs = std::filesystem; disk_dir = dir; disk_limit = 1024ull*1024ull*(size_t) std::max(limit_mib, 0); disk_key = fnv1a(key.data(), key.size());+ disk_media = media.empty() ? 0 : std::max<uint64_t>(1, fnv1a(media.data(), media.size())); std::error_code ec; fs::create_directories(dir, ec);@@ -3029,16 +3090,16 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, if (ext != ".kvc") { continue; }- llama_tokens tokens;+ server_tokens tokens; uint32_t version = 0; cache_meta meta; bool ok; { disk_file f(de.path().string(), "rb");- ok = f.f && read_header(f.f, disk_key, tokens, version, meta);+ ok = f.f && read_header(f.f, disk_key, disk_media, tokens, version, meta); } if (!ok) {- // another model, cache layout or format+ // another model, cache layout, format or multimodal projector fs::remove(de.path(), ec); continue; }@@ -3051,14 +3112,15 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, meta.id = new_entry_id(); meta.created = t; meta.last_used = t;- meta.family = fnv1a(tokens.data(), std::min(tokens.size(), FAMILY_TOKENS) * sizeof(llama_token));+ const llama_tokens raw = raw_tokens(tokens);+ meta.family = fnv1a(raw.data(), std::min(raw.size(), FAMILY_TOKENS) * sizeof(llama_token)); if (detokenize) {- meta.excerpt = user_excerpt(detokenize(llama_tokens(tokens.begin(), tokens.begin() + std::min(tokens.size(), CLASSIFY_HEAD))));+ meta.excerpt = user_excerpt(detokenize(llama_tokens(raw.begin(), raw.begin() + std::min(raw.size(), CLASSIFY_HEAD)))); } set_expiry(meta); n_v1++; }- found.push_back({ mtime, { de.path().string(), std::move(tokens), (size_t) de.file_size(ec), std::move(meta), version } });+ found.emplace_back(mtime, disk_entry { de.path().string(), std::move(tokens), (size_t) de.file_size(ec), std::move(meta), version }); } std::sort(found.begin(), found.end(), [](const auto & a, const auto & b) { return a.first < b.first; }); @@ -3073,17 +3135,21 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, total += e.size; } - SRV_INF("prompt cache: disk tier %s, %zu prompts (%zu v1), %.1f GiB (limit %.1f GiB)\n",- dir.c_str(), disk.size(), n_v1, total / (1024.0*1024.0*1024.0), disk_limit / (1024.0*1024.0*1024.0));+ size_t n_media = 0;+ for (const auto & e : disk) {+ n_media += e.version == DISK_VERSION_MEDIA;+ }++ SRV_INF("prompt cache: disk tier %s, %zu prompts (%zu v1, %zu with images), %.1f GiB (limit %.1f GiB)\n",+ dir.c_str(), disk.size(), n_v1, n_media, total / (1024.0*1024.0*1024.0), disk_limit / (1024.0*1024.0*1024.0)); disk_thread = std::thread([this] { disk_write_loop(); }); } void server_prompt_cache::evict(std::list<server_prompt_cache_state>::iterator it) { if (disk_thread.joinable() && !expired(it->prompt.meta, unix_now())) {- // text prompts only: media chunks are not serialized- const llama_tokens tokens = raw_tokens(it->prompt.tokens);- if (std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) == tokens.end()) {+ // a prompt with images needs the media key its file is read back under+ if (disk_media != 0 || !has_media(it->prompt.tokens)) { std::lock_guard<std::mutex> lock(disk_mutex); pending.splice(pending.end(), states, it); disk_cv.notify_all();@@ -3131,8 +3197,20 @@ void server_prompt_cache::disk_write_loop() { namespace { -// the whole file at path (via path.tmp): header, metadata, then the v1 body-bool write_file(const std::string & path, uint64_t key, const llama_tokens & tokens, const server_prompt_cache_state & state, const cache_meta & meta) {+// what a file stores as its token list: the tokens of a text prompt, server_tokens::serialize() of one with media+llama_tokens disk_tokens(const server_tokens & t) {+ if (!has_media(t)) {+ return raw_tokens(t);+ }+ const std::vector<char> blob = t.serialize();+ llama_tokens res(blob.size() / sizeof(llama_token));+ memcpy(res.data(), blob.data(), res.size() * sizeof(llama_token));+ return res;+}++// the whole file at path (via path.tmp): header, metadata, then the v1 body. media != 0 writes v3: the media key+// before the token list, which then holds the serialized media chunks too+bool write_file(const std::string & path, uint64_t key, uint64_t media, const llama_tokens & tokens, const server_prompt_cache_state & state, const cache_meta & meta) { const std::string tmp = path + ".tmp"; bool ok; {@@ -3141,7 +3219,8 @@ bool write_file(const std::string & path, uint64_t key, const llama_tokens & tok if (ok) { setvbuf(f.f, nullptr, _IOFBF, 8u << 20); const auto mb = meta_encode(meta);- ok = put(f.f, DISK_MAGIC) && put(f.f, DISK_VERSION) && put(f.f, key) && put(f.f, mb.data(), mb.size()) &&+ ok = put(f.f, DISK_MAGIC) && put(f.f, media != 0 ? DISK_VERSION_MEDIA : DISK_VERSION) && put(f.f, key) &&+ put(f.f, mb.data(), mb.size()) && (media == 0 || put(f.f, media)) && put(f.f, (uint64_t) tokens.size()) && put(f.f, tokens.data(), tokens.size() * sizeof(llama_token)) && put(f.f, (uint64_t) state.prompt.checkpoints.size()); for (const auto & ckpt : state.prompt.checkpoints) {@@ -3166,7 +3245,7 @@ bool write_file(const std::string & path, uint64_t key, const llama_tokens & tok } // namespace void server_prompt_cache::disk_rewrite_meta_locked(disk_entry & e) {- if (e.version == DISK_VERSION) {+ if (e.version >= DISK_VERSION) { if (!write_meta(e.path, e.meta)) { SRV_WRN("prompt cache: cannot update metadata of %s\n", e.path.c_str()); }@@ -3179,20 +3258,20 @@ void server_prompt_cache::disk_rewrite_meta_locked(disk_entry & e) { // rewrites a v1 file as v2 so its metadata can be stored void server_prompt_cache::disk_upgrade(const std::string & path) { server_prompt_cache_state state;- if (!disk_read(path, false, state)) {+ if (!disk_read(path, state)) { return; } cache_meta meta; { std::lock_guard<std::mutex> lock(disk_mutex); auto it = std::find_if(disk.begin(), disk.end(), [&](const disk_entry & e) { return e.path == path; });- if (it == disk.end() || it->version == DISK_VERSION) {+ if (it == disk.end() || it->version != 1) { return; } meta = it->meta; }- const llama_tokens tokens = raw_tokens(state.prompt.tokens);- const bool ok = write_file(path, disk_key, tokens, state, meta);+ // v1 files hold text prompts only+ const bool ok = write_file(path, disk_key, 0, raw_tokens(state.prompt.tokens), state, meta); std::lock_guard<std::mutex> lock(disk_mutex); auto it = std::find_if(disk.begin(), disk.end(), [&](const disk_entry & e) { return e.path == path; });@@ -3267,7 +3346,8 @@ void server_prompt_cache::disk_evict_locked(const std::string & newest) { void server_prompt_cache::disk_write(server_prompt_cache_state & state) { namespace fs = std::filesystem; - const llama_tokens tokens = raw_tokens(state.prompt.tokens);+ const server_tokens & tokens = state.prompt.tokens;+ const bool media = has_media(tokens); cache_meta meta; { std::lock_guard<std::mutex> lock(disk_mutex);@@ -3285,10 +3365,12 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) { } const int64_t t_start = ggml_time_us();- const uint64_t h = fnv1a(tokens.data(), tokens.size() * sizeof(llama_token));+ // the name hashes the media chunks too: prompts with equal text and other images get files of their own+ const llama_tokens words = disk_tokens(tokens);+ const uint64_t h = fnv1a(words.data(), words.size() * sizeof(llama_token)); const std::string path = string_format("%s/%016llx-%zu.kvc", disk_dir.c_str(), (unsigned long long) h, tokens.size()); - if (!write_file(path, disk_key, tokens, state, meta)) {+ if (!write_file(path, disk_key, media ? disk_media : 0, words, state, meta)) { SRV_ERR("prompt cache: failed to write %s\n", path.c_str()); return; }@@ -3311,7 +3393,7 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) { } } disk.remove_if([&](const disk_entry & e) { return e.path == path; });- disk.push_back({ path, tokens, size, latest, DISK_VERSION });+ disk.push_back({ path, tokens.clone(), size, latest, media ? DISK_VERSION_MEDIA : DISK_VERSION }); if (meta_encode(latest) != meta_encode(meta)) { set_expiry(disk.back().meta); write_meta(path, disk.back().meta);@@ -3324,8 +3406,8 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) { total += e.size; } - SRV_INF("prompt cache: wrote %zu tokens (%s, %s) to disk, %.1f MiB in %.2f s (%zu prompts, %.1f GiB)\n",- tokens.size(), hex_id(meta.id).c_str(), cache_class_name(meta.cls), size / (1024.0*1024.0), (ggml_time_us() - t_start) / 1e6,+ SRV_INF("prompt cache: wrote %zu tokens, %lld images (%s, %s) to disk, %.1f MiB in %.2f s (%zu prompts, %.1f GiB)\n",+ tokens.size(), (long long) std::max<int64_t>(0, media_count(tokens)), hex_id(meta.id).c_str(), cache_class_name(meta.cls), size / (1024.0*1024.0), (ggml_time_us() - t_start) / 1e6, disk.size(), total / (1024.0*1024.0*1024.0)); } @@ -3336,7 +3418,7 @@ json server_prompt_cache::info() { { tier_report rep; for (const auto & s : states) {- rep.add(s.prompt.meta, s.prompt.tokens.size(), s.size(), now);+ rep.add(s.prompt.meta, s.prompt.tokens.size(), std::max<int64_t>(0, media_count(s.prompt.tokens)), s.size(), now); } rep.to_json(ram); }@@ -3359,15 +3441,18 @@ json server_prompt_cache::info() { std::lock_guard<std::mutex> lock(disk_mutex); tier_report rep; size_t n_v1 = 0;+ size_t n_media = 0; for (const auto & e : disk) {- rep.add(e.meta, e.tokens.size(), e.size, now);+ rep.add(e.meta, e.tokens.size(), std::max<int64_t>(0, media_count(e.tokens)), e.size, now); n_v1 += e.version == 1;+ n_media += e.version == DISK_VERSION_MEDIA; } json d = { { "dir", disk_dir }, { "limit_bytes", disk_limit }, { "pending", pending.size() }, { "v1_files", n_v1 },+ { "image_files", n_media }, }; rep.to_json(d); res["disk"] = d;@@ -3396,18 +3481,18 @@ size_t server_prompt_cache::clear_disk(bool include_pinned) { return n; } -bool server_prompt_cache::disk_read(const std::string & path, bool has_mtmd, server_prompt_cache_state & out) const {+bool server_prompt_cache::disk_read(const std::string & path, server_prompt_cache_state & out) const { disk_file f(path, "rb"); if (!f.f) { return false; } setvbuf(f.f, nullptr, _IOFBF, 8u << 20); - llama_tokens tokens;+ server_tokens tokens; uint32_t version; cache_meta meta; uint64_t n_ckpt;- if (!read_header(f.f, disk_key, tokens, version, meta) || !get(f.f, n_ckpt) || n_ckpt > 4096) {+ if (!read_header(f.f, disk_key, disk_media, tokens, version, meta) || !get(f.f, n_ckpt) || n_ckpt > 4096) { return false; } std::list<common_prompt_checkpoint> checkpoints;@@ -3425,7 +3510,7 @@ bool server_prompt_cache::disk_read(const std::string & path, bool has_mtmd, ser return false; } - out.prompt.tokens = server_tokens(tokens, has_mtmd);+ out.prompt.tokens = std::move(tokens); out.prompt.checkpoints = std::move(checkpoints); out.prompt.meta = std::move(meta); out.data = std::move(data);diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex 21554dfe..2575c9cf 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -741,17 +741,19 @@ struct server_prompt_cache { bool load_entry(std::list<server_prompt_cache_state>::iterator it, server_prompt & prompt, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot, bool keep_entry); // the disk entry whose tokens score highest, above n_best (which it raises to that score); empty if none- std::string disk_best(const std::function<int32_t(const llama_tokens &)> & score, int32_t & n_best);+ std::string disk_best(const std::function<int32_t(const server_tokens &)> & score, int32_t & n_best); // reads a disk entry into RAM and returns it, or states.end() if the file cannot be read (it is then removed)- std::list<server_prompt_cache_state>::iterator disk_promote(const std::string & path, bool has_mtmd);+ std::list<server_prompt_cache_state>::iterator disk_promote(const std::string & path); void update(); // second tier on disk: an entry the RAM limit evicts is written to dir instead of dropped, and read back // when it is the best prefix match for a new prompt; flush() writes what is still in RAM (on shutdown).- // key names the model and cache layout; files written under another key are deleted. text prompts only.- void disk_init(const std::string & dir, int32_t limit_mib, const std::string & key);+ // key names the model and cache layout; files written under another key are deleted. media names the+ // multimodal projector and its image settings (empty without one): a prompt with images is stored with its+ // media chunk map (v3) and only read back under the same media key.+ void disk_init(const std::string & dir, int32_t limit_mib, const std::string & key, const std::string & media); void flush(); @@ -790,11 +792,11 @@ struct server_prompt_cache { private: struct disk_entry {- std::string path;- llama_tokens tokens;- size_t size;- cache_meta meta;- uint32_t version;+ std::string path;+ server_tokens tokens; // media chunks are placeholders+ size_t size;+ cache_meta meta;+ uint32_t version; }; struct family_stat {@@ -820,6 +822,7 @@ private: std::string disk_dir; size_t disk_limit = 0; uint64_t disk_key = 0;+ uint64_t disk_media = 0; // 0 = no multimodal projector // oldest first; the lists and stop are guarded by disk_mutex std::list<disk_entry> disk;@@ -860,7 +863,7 @@ private: void disk_write_loop(); void disk_write(server_prompt_cache_state & state); void disk_upgrade(const std::string & path);- bool disk_read(const std::string & path, bool has_mtmd, server_prompt_cache_state & out) const;+ bool disk_read(const std::string & path, server_prompt_cache_state & out) const; void disk_evict_locked(const std::string & newest); void disk_rewrite_meta_locked(disk_entry & e);