tjh.dev
1{
2 lib,
3 buildPythonPackage,
4 hatch-fancy-pypi-readme,
5 hatch-vcs,
6 hatchling,
7 pytestCheckHook,
8 pythonOlder,
9 accelerate,
10 bentoml,
11 bitsandbytes,
12 build,
13 click,
14 ctranslate2,
15 datasets,
16 docker,
17 einops,
18 ghapi,
19 huggingface-hub,
20 hypothesis,
21 ipython,
22 jupyter,
23 jupytext,
24 nbformat,
25 notebook,
26 openai,
27 openllm-client,
28 openllm-core,
29 optimum,
30 peft,
31 pytest-mock,
32 pytest-randomly,
33 pytest-rerunfailures,
34 pytest-xdist,
35 safetensors,
36 scipy,
37 sentencepiece,
38 soundfile,
39 syrupy,
40 tabulate,
41 tiktoken,
42 transformers,
43 triton,
44 xformers,
45}:
46
47buildPythonPackage rec {
48 inherit (openllm-core) src version;
49 pname = "openllm";
50 pyproject = true;
51
52 disabled = pythonOlder "3.8";
53
54 sourceRoot = "${src.name}/openllm-python";
55
56
57 pythonRemoveDeps = [
58 # remove cuda-python as it has an unfree license
59 "cuda-python"
60 ];
61
62 build-system = [
63 hatch-fancy-pypi-readme
64 hatch-vcs
65 hatchling
66 ];
67
68 dependencies =
69 [
70 accelerate
71 bentoml
72 bitsandbytes
73 build
74 click
75 einops
76 ghapi
77 openllm-client
78 openllm-core
79 optimum
80 safetensors
81 scipy
82 sentencepiece
83 transformers
84 ]
85 ++ bentoml.optional-dependencies.io
86 ++ tabulate.optional-dependencies.widechars
87 ++ transformers.optional-dependencies.tokenizers
88 ++ transformers.optional-dependencies.torch;
89
90 optional-dependencies = {
91 agents = [
92 # diffusers
93 soundfile
94 transformers
95 ] ++ transformers.optional-dependencies.agents;
96 awq = [
97 # autoawq
98 ];
99 baichuan = [
100 # cpm-kernels
101 ];
102 chatglm = [
103 # cpm-kernels
104 ];
105 ctranslate = [ ctranslate2 ];
106 falcon = [ xformers ];
107 fine-tune = [
108 datasets
109 huggingface-hub
110 peft
111 # trl
112 ];
113 ggml = [
114 # ctransformers
115 ];
116 gptq = [
117 # auto-gptq
118 ]; # ++ autogptq.optional-dependencies.triton;
119 grpc = [ bentoml ] ++ bentoml.optional-dependencies.grpc;
120 mpt = [ triton ];
121 openai = [
122 openai
123 tiktoken
124 ] ++ openai.optional-dependencies.datalib;
125 playground = [
126 ipython
127 jupyter
128 jupytext
129 nbformat
130 notebook
131 ];
132 starcoder = [ bitsandbytes ];
133 vllm = [
134 # vllm
135 ];
136 full =
137 with optional-dependencies;
138 (
139 agents
140 ++ awq
141 ++ baichuan
142 ++ chatglm
143 ++ ctranslate
144 ++ falcon
145 ++ fine-tune
146 ++ ggml
147 ++ gptq
148 ++ mpt
149 # disambiguate between derivation input and passthru field
150 ++ optional-dependencies.openai
151 ++ playground
152 ++ starcoder
153 ++ vllm
154 );
155 all = optional-dependencies.full;
156 };
157
158 nativeCheckInputs = [
159 docker
160 hypothesis
161 pytest-mock
162 pytest-randomly
163 pytest-rerunfailures
164 pytest-xdist
165 pytestCheckHook
166 syrupy
167 ];
168
169 preCheck = ''
170 export HOME=$TMPDIR
171 # skip GPUs test on CI
172 export GITHUB_ACTIONS=1
173 # disable hypothesis' deadline
174 export CI=1
175 '';
176
177 disabledTestPaths = [
178 # require network access
179 "tests/models"
180 ];
181
182 disabledTests = [
183 # incompatible with recent TypedDict
184 # https://github.com/bentoml/OpenLLM/blob/f3fd32d596253ae34c68e2e9655f19f40e05f666/openllm-python/tests/configuration_test.py#L18-L21
185 "test_missing_default"
186 ];
187
188 pythonImportsCheck = [ "openllm" ];
189
190 meta = with lib; {
191 description = "Operating LLMs in production";
192 homepage = "https://github.com/bentoml/OpenLLM/tree/main/openllm-python";
193 changelog = "https://github.com/bentoml/OpenLLM/blob/${src.rev}/CHANGELOG.md";
194 license = licenses.asl20;
195 maintainers = with maintainers; [
196 happysalada
197 natsukium
198 ];
199 };
200}