tjh.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5
6 # build-system
7 setuptools,
8
9 # dependencies
10 filelock,
11 huggingface-hub,
12 numpy,
13 protobuf,
14 packaging,
15 pyyaml,
16 regex,
17 requests,
18 tokenizers,
19 safetensors,
20 tqdm,
21
22 # optional-dependencies
23 diffusers,
24 scikit-learn,
25 tensorflow,
26 onnxconverter-common,
27 opencv4,
28 tf2onnx,
29 torch,
30 accelerate,
31 faiss,
32 datasets,
33 jax,
34 jaxlib,
35 flax,
36 optax,
37 ftfy,
38 onnxruntime,
39 onnxruntime-tools,
40 cookiecutter,
41 sagemaker,
42 fairscale,
43 optuna,
44 ray,
45 pydantic,
46 uvicorn,
47 fastapi,
48 starlette,
49 librosa,
50 phonemizer,
51 torchaudio,
52 pillow,
53 timm,
54 torchvision,
55 av,
56 sentencepiece,
57}:
58
59buildPythonPackage rec {
60 pname = "transformers";
61 version = "4.47.1";
62 pyproject = true;
63
64 src = fetchFromGitHub {
65 owner = "huggingface";
66 repo = "transformers";
67 tag = "v${version}";
68 hash = "sha256-xwc84wFUSRJ8SNCLiI7FQ1v/JKnXkTW4EpNCjgUbZ8E=";
69 };
70
71 build-system = [ setuptools ];
72
73 dependencies = [
74 filelock
75 huggingface-hub
76 numpy
77 packaging
78 pyyaml
79 regex
80 requests
81 tokenizers
82 safetensors
83 tqdm
84 ];
85
86 optional-dependencies =
87 let
88 audio = [
89 librosa
90 # pyctcdecode
91 phonemizer
92 # kenlm
93 ];
94 vision = [ pillow ];
95 in
96 {
97 agents = [
98 diffusers
99 accelerate
100 datasets
101 torch
102 sentencepiece
103 opencv4
104 pillow
105 ];
106 ja = [
107 # fugashi
108 # ipadic
109 # rhoknp
110 # sudachidict_core
111 # sudachipy
112 # unidic
113 # unidic_lite
114 ];
115 sklearn = [ scikit-learn ];
116 tf = [
117 tensorflow
118 onnxconverter-common
119 tf2onnx
120 # tensorflow-text
121 # keras-nlp
122 ];
123 torch = [
124 torch
125 accelerate
126 ];
127 retrieval = [
128 faiss
129 datasets
130 ];
131 flax = [
132 jax
133 jaxlib
134 flax
135 optax
136 ];
137 tokenizers = [ tokenizers ];
138 ftfy = [ ftfy ];
139 onnxruntime = [
140 onnxruntime
141 onnxruntime-tools
142 ];
143 onnx = [
144 onnxconverter-common
145 tf2onnx
146 onnxruntime
147 onnxruntime-tools
148 ];
149 modelcreation = [ cookiecutter ];
150 sagemaker = [ sagemaker ];
151 deepspeed = [
152 # deepspeed
153 accelerate
154 ];
155 fairscale = [ fairscale ];
156 optuna = [ optuna ];
157 ray = [ ray ] ++ ray.optional-dependencies.tune;
158 # sigopt = [ sigopt ];
159 # integrations = ray ++ optuna ++ sigopt;
160 serving = [
161 pydantic
162 uvicorn
163 fastapi
164 starlette
165 ];
166 audio = audio;
167 speech = [ torchaudio ] ++ audio;
168 torch-speech = [ torchaudio ] ++ audio;
169 tf-speech = audio;
170 flax-speech = audio;
171 timm = [ timm ];
172 torch-vision = [ torchvision ] ++ vision;
173 # natten = [ natten ];
174 # codecarbon = [ codecarbon ];
175 video = [
176 av
177 ];
178 sentencepiece = [
179 sentencepiece
180 protobuf
181 ];
182 };
183
184 # Many tests require internet access.
185 doCheck = false;
186
187 pythonImportsCheck = [ "transformers" ];
188
189 meta = {
190 homepage = "https://github.com/huggingface/transformers";
191 description = "Natural Language Processing for TensorFlow 2.0 and PyTorch";
192 mainProgram = "transformers-cli";
193 changelog = "https://github.com/huggingface/transformers/releases/tag/${src.tag}";
194 license = lib.licenses.asl20;
195 platforms = lib.platforms.unix;
196 maintainers = with lib.maintainers; [
197 pashashocky
198 happysalada
199 ];
200 };
201}