pyrox.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5
6 # build-system
7 setuptools,
8
9 # dependencies
10 filelock,
11 huggingface-hub,
12 numpy,
13 protobuf,
14 packaging,
15 pyyaml,
16 regex,
17 requests,
18 tokenizers,
19 safetensors,
20 tqdm,
21
22 # optional-dependencies
23 diffusers,
24 scikit-learn,
25 tensorflow,
26 onnxconverter-common,
27 opencv4,
28 tf2onnx,
29 torch,
30 accelerate,
31 faiss,
32 datasets,
33 jax,
34 jaxlib,
35 flax,
36 optax,
37 ftfy,
38 onnxruntime,
39 onnxruntime-tools,
40 cookiecutter,
41 sagemaker,
42 fairscale,
43 optuna,
44 ray,
45 pydantic,
46 uvicorn,
47 fastapi,
48 starlette,
49 librosa,
50 phonemizer,
51 torchaudio,
52 pillow,
53 timm,
54 torchvision,
55 av,
56 sentencepiece,
57}:
58
59buildPythonPackage rec {
60 pname = "transformers";
61 version = "4.51.3";
62 pyproject = true;
63
64 src = fetchFromGitHub {
65 owner = "huggingface";
66 repo = "transformers";
67 tag = "v${version}";
68 hash = "sha256-VYEkBt0fpG27MVdnABcMAMmk2Pzsc+2Fetx/GmeFBac=";
69 };
70
71 build-system = [ setuptools ];
72
73 dependencies = [
74 filelock
75 huggingface-hub
76 numpy
77 packaging
78 pyyaml
79 regex
80 requests
81 tokenizers
82 safetensors
83 tqdm
84 ];
85
86 optional-dependencies =
87 let
88 audio = [
89 librosa
90 # pyctcdecode
91 phonemizer
92 # kenlm
93 ];
94 vision = [ pillow ];
95 in
96 {
97 agents = [
98 diffusers
99 accelerate
100 datasets
101 torch
102 sentencepiece
103 opencv4
104 pillow
105 ];
106 ja = [
107 # fugashi
108 # ipadic
109 # rhoknp
110 # sudachidict_core
111 # sudachipy
112 # unidic
113 # unidic_lite
114 ];
115 sklearn = [ scikit-learn ];
116 tf = [
117 tensorflow
118 onnxconverter-common
119 tf2onnx
120 # tensorflow-text
121 # keras-nlp
122 ];
123 torch = [
124 torch
125 accelerate
126 ];
127 retrieval = [
128 faiss
129 datasets
130 ];
131 flax = [
132 jax
133 jaxlib
134 flax
135 optax
136 ];
137 hf_xet = [
138 # hf-xet (unpackaged)
139 ];
140 tokenizers = [ tokenizers ];
141 ftfy = [ ftfy ];
142 onnxruntime = [
143 onnxruntime
144 onnxruntime-tools
145 ];
146 onnx = [
147 onnxconverter-common
148 tf2onnx
149 onnxruntime
150 onnxruntime-tools
151 ];
152 modelcreation = [ cookiecutter ];
153 sagemaker = [ sagemaker ];
154 deepspeed = [
155 # deepspeed
156 accelerate
157 ];
158 fairscale = [ fairscale ];
159 optuna = [ optuna ];
160 ray = [ ray ] ++ ray.optional-dependencies.tune;
161 # sigopt = [ sigopt ];
162 # integrations = ray ++ optuna ++ sigopt;
163 serving = [
164 pydantic
165 uvicorn
166 fastapi
167 starlette
168 ];
169 audio = audio;
170 speech = [ torchaudio ] ++ audio;
171 torch-speech = [ torchaudio ] ++ audio;
172 tf-speech = audio;
173 flax-speech = audio;
174 timm = [ timm ];
175 torch-vision = [ torchvision ] ++ vision;
176 # natten = [ natten ];
177 # codecarbon = [ codecarbon ];
178 video = [
179 av
180 ];
181 sentencepiece = [
182 sentencepiece
183 protobuf
184 ];
185 };
186
187 # Many tests require internet access.
188 doCheck = false;
189
190 pythonImportsCheck = [ "transformers" ];
191
192 meta = {
193 homepage = "https://github.com/huggingface/transformers";
194 description = "Natural Language Processing for TensorFlow 2.0 and PyTorch";
195 mainProgram = "transformers-cli";
196 changelog = "https://github.com/huggingface/transformers/releases/tag/v${version}";
197 license = lib.licenses.asl20;
198 platforms = lib.platforms.unix;
199 maintainers = with lib.maintainers; [
200 pashashocky
201 happysalada
202 ];
203 };
204}