tjh.dev
1{
2 pkgs,
3 config,
4 lib,
5 ...
6}:
7let
8 cfg = config.services.local-ai;
9 inherit (lib) mkOption types;
10in
11{
12 options.services.local-ai = {
13 enable = lib.mkEnableOption "Enable service";
14
15 package = lib.mkPackageOption pkgs "local-ai" { };
16
17 extraArgs = mkOption {
18 type = types.listOf types.str;
19 default = [ ];
20 };
21
22 port = mkOption {
23 type = types.port;
24 default = 8080;
25 };
26
27 threads = mkOption {
28 type = types.int;
29 default = 1;
30 };
31
32 models = mkOption {
33 type = types.either types.package types.str;
34 default = "models";
35 };
36
37 parallelRequests = mkOption {
38 type = types.int;
39 default = 1;
40 };
41
42 logLevel = mkOption {
43 type = types.enum [
44 "error"
45 "warn"
46 "info"
47 "debug"
48 "trace"
49 ];
50 default = "warn";
51 };
52 };
53
54 config = lib.mkIf cfg.enable {
55 systemd.services.local-ai = {
56 wantedBy = [ "multi-user.target" ];
57 environment.LLAMACPP_PARALLEL = toString cfg.parallelRequests;
58 serviceConfig = {
59 DynamicUser = true;
60 ExecStart = lib.escapeShellArgs (
61 [
62 "${cfg.package}/bin/local-ai"
63 "--address=:${toString cfg.port}"
64 "--threads=${toString cfg.threads}"
65 "--localai-config-dir=."
66 "--models-path=${cfg.models}"
67 "--log-level=${cfg.logLevel}"
68 ]
69 ++ lib.optional (cfg.parallelRequests > 1) "--parallel-requests"
70 ++ cfg.extraArgs
71 );
72 RuntimeDirectory = "local-ai";
73 WorkingDirectory = "%t/local-ai";
74 };
75 };
76 };
77}