pyrox.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5 pythonOlder,
6 setuptools,
7 # propagated build inputs
8 filelock,
9 huggingface-hub,
10 numpy,
11 protobuf,
12 packaging,
13 pyyaml,
14 regex,
15 requests,
16 tokenizers,
17 safetensors,
18 tqdm,
19 # optional dependencies
20 diffusers,
21 scikit-learn,
22 tensorflow,
23 onnxconverter-common,
24 opencv4,
25 tf2onnx,
26 torch,
27 accelerate,
28 faiss,
29 datasets,
30 jax,
31 jaxlib,
32 flax,
33 optax,
34 ftfy,
35 onnxruntime,
36 onnxruntime-tools,
37 cookiecutter,
38 sagemaker,
39 fairscale,
40 optuna,
41 ray,
42 pydantic,
43 uvicorn,
44 fastapi,
45 starlette,
46 librosa,
47 phonemizer,
48 torchaudio,
49 pillow,
50 timm,
51 torchvision,
52 av,
53 sentencepiece,
54}:
55
56buildPythonPackage rec {
57 pname = "transformers";
58 version = "4.41.0";
59 pyproject = true;
60
61 disabled = pythonOlder "3.8";
62
63 src = fetchFromGitHub {
64 owner = "huggingface";
65 repo = "transformers";
66 rev = "refs/tags/v${version}";
67 hash = "sha256-FUYQeEksjDasFvQraycNFAx3cLHfDdPpgZssqN8OIJw=";
68 };
69
70 build-system = [ setuptools ];
71
72 dependencies = [
73 filelock
74 huggingface-hub
75 numpy
76 packaging
77 pyyaml
78 regex
79 requests
80 tokenizers
81 safetensors
82 tqdm
83 ];
84
85 passthru.optional-dependencies =
86 let
87 audio = [
88 librosa
89 # pyctcdecode
90 phonemizer
91 # kenlm
92 ];
93 vision = [ pillow ];
94 in
95 {
96 agents = [
97 diffusers
98 accelerate
99 datasets
100 torch
101 sentencepiece
102 opencv4
103 pillow
104 ];
105 ja = [
106 # fugashi
107 # ipadic
108 # rhoknp
109 # sudachidict_core
110 # sudachipy
111 # unidic
112 # unidic_lite
113 ];
114 sklearn = [ scikit-learn ];
115 tf = [
116 tensorflow
117 onnxconverter-common
118 tf2onnx
119 # tensorflow-text
120 # keras-nlp
121 ];
122 torch = [
123 torch
124 accelerate
125 ];
126 retrieval = [
127 faiss
128 datasets
129 ];
130 flax = [
131 jax
132 jaxlib
133 flax
134 optax
135 ];
136 tokenizers = [ tokenizers ];
137 ftfy = [ ftfy ];
138 onnxruntime = [
139 onnxruntime
140 onnxruntime-tools
141 ];
142 onnx = [
143 onnxconverter-common
144 tf2onnx
145 onnxruntime
146 onnxruntime-tools
147 ];
148 modelcreation = [ cookiecutter ];
149 sagemaker = [ sagemaker ];
150 deepspeed = [
151 # deepspeed
152 accelerate
153 ];
154 fairscale = [ fairscale ];
155 optuna = [ optuna ];
156 ray = [ ray ] ++ ray.optional-dependencies.tune-deps;
157 # sigopt = [ sigopt ];
158 # integrations = ray ++ optuna ++ sigopt;
159 serving = [
160 pydantic
161 uvicorn
162 fastapi
163 starlette
164 ];
165 audio = audio;
166 speech = [ torchaudio ] ++ audio;
167 torch-speech = [ torchaudio ] ++ audio;
168 tf-speech = audio;
169 flax-speech = audio;
170 timm = [ timm ];
171 torch-vision = [ torchvision ] ++ vision;
172 # natten = [ natten ];
173 # codecarbon = [ codecarbon ];
174 video = [
175 # decord
176 av
177 ];
178 sentencepiece = [
179 sentencepiece
180 protobuf
181 ];
182 };
183
184 # Many tests require internet access.
185 doCheck = false;
186
187 pythonImportsCheck = [ "transformers" ];
188
189 meta = with lib; {
190 homepage = "https://github.com/huggingface/transformers";
191 description = "Natural Language Processing for TensorFlow 2.0 and PyTorch";
192 mainProgram = "transformers-cli";
193 changelog = "https://github.com/huggingface/transformers/releases/tag/v${version}";
194 license = licenses.asl20;
195 platforms = platforms.unix;
196 maintainers = with maintainers; [
197 pashashocky
198 happysalada
199 ];
200 };
201}