pyrox.dev
1{ lib
2, buildPythonPackage
3, fetchFromGitHub
4, pythonOlder
5# propagated build inputs
6, filelock
7, huggingface-hub
8, numpy
9, protobuf
10, packaging
11, pyyaml
12, regex
13, requests
14, tokenizers
15, safetensors
16, tqdm
17# optional dependencies
18, scikit-learn
19, tensorflow
20, onnxconverter-common
21, tf2onnx
22, torch
23, accelerate
24, faiss
25, datasets
26, jax
27, jaxlib
28, flax
29, optax
30, ftfy
31, onnxruntime
32, onnxruntime-tools
33, cookiecutter
34, sagemaker
35, fairscale
36, optuna
37, ray
38, pydantic
39, uvicorn
40, fastapi
41, starlette
42, librosa
43, phonemizer
44, torchaudio
45, pillow
46, timm
47, torchvision
48, av
49, sentencepiece
50}:
51
52buildPythonPackage rec {
53 pname = "transformers";
54 version = "4.35.2";
55 format = "setuptools";
56
57 disabled = pythonOlder "3.8";
58
59 src = fetchFromGitHub {
60 owner = "huggingface";
61 repo = "transformers";
62 rev = "refs/tags/v${version}";
63 hash = "sha256-h1RMSEcuali05AWeTm1wyZQJz6XrHamCF1eHrSnFnfM=";
64 };
65
66 propagatedBuildInputs = [
67 filelock
68 huggingface-hub
69 numpy
70 packaging
71 pyyaml
72 regex
73 requests
74 tokenizers
75 safetensors
76 tqdm
77 ];
78
79 passthru.optional-dependencies =
80 let
81 audio = [
82 librosa
83 # pyctcdecode
84 phonemizer
85 # kenlm
86 ];
87 vision = [ pillow ];
88 in
89 {
90 ja = [
91 # fugashi
92 # ipadic
93 # rhoknp
94 # sudachidict_core
95 # sudachipy
96 # unidic
97 # unidic_lite
98 ];
99 sklearn = [
100 scikit-learn
101 ];
102 tf = [
103 tensorflow
104 onnxconverter-common
105 tf2onnx
106 # tensorflow-text
107 # keras-nlp
108 ];
109 torch = [
110 torch
111 accelerate
112 ];
113 retrieval = [ faiss datasets ];
114 flax = [ jax jaxlib flax optax ];
115 tokenizers = [
116 tokenizers
117 ];
118 ftfy = [ ftfy ];
119 onnxruntime = [
120 onnxruntime
121 onnxruntime-tools
122 ];
123 onnx = [
124 onnxconverter-common
125 tf2onnx
126 onnxruntime
127 onnxruntime-tools
128 ];
129 modelcreation = [
130 cookiecutter
131 ];
132 sagemaker = [
133 sagemaker
134 ];
135 deepspeed = [
136 # deepspeed
137 accelerate
138 ];
139 fairscale = [ fairscale ];
140 optuna = [ optuna ];
141 ray = [ ray ] ++ ray.optional-dependencies.tune-deps;
142 # sigopt = [ sigopt ];
143 # integrations = ray ++ optuna ++ sigopt;
144 serving = [
145 pydantic
146 uvicorn
147 fastapi
148 starlette
149 ];
150 audio = audio;
151 speech = [ torchaudio ] ++ audio;
152 torch-speech = [ torchaudio ] ++ audio;
153 tf-speech = audio;
154 flax-speech = audio;
155 timm = [ timm ];
156 torch-vision = [ torchvision ] ++ vision;
157 # natten = [ natten ];
158 # codecarbon = [ codecarbon ];
159 video = [
160 # decord
161 av
162 ];
163 sentencepiece = [ sentencepiece protobuf ];
164 };
165
166
167 # Many tests require internet access.
168 doCheck = false;
169
170 pythonImportsCheck = [
171 "transformers"
172 ];
173
174 meta = with lib; {
175 homepage = "https://github.com/huggingface/transformers";
176 description = "Natural Language Processing for TensorFlow 2.0 and PyTorch";
177 changelog = "https://github.com/huggingface/transformers/releases/tag/v${version}";
178 license = licenses.asl20;
179 platforms = platforms.unix;
180 maintainers = with maintainers; [ pashashocky happysalada ];
181 };
182}