tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{ lib
2, stdenv
3, python
4, buildPythonPackage
5, fetchFromGitHub
6, alembic
7, argcomplete
8, attrs
9, blinker
10, cached-property
11, cattrs
12, clickclick
13, colorlog
14, croniter
15, cryptography
16, dataclasses
17, dill
18, flask
19, flask_login
20, flask_wtf
21, flask-appbuilder
22, flask-caching
23, GitPython
24, graphviz
25, gunicorn
26, httpx
27, iso8601
28, importlib-resources
29, importlib-metadata
30, inflection
31, itsdangerous
32, jinja2
33, jsonschema
34, lazy-object-proxy
35, lockfile
36, markdown
37, markupsafe
38, marshmallow-oneofschema
39, numpy
40, openapi-spec-validator
41, pandas
42, pendulum
43, psutil
44, pygments
45, pyjwt
46, python-daemon
47, python-dateutil
48, python-nvd3
49, python-slugify
50, python3-openid
51, pythonOlder
52, pyyaml
53, rich
54, setproctitle
55, sqlalchemy
56, sqlalchemy-jsonfield
57, swagger-ui-bundle
58, tabulate
59, tenacity
60, termcolor
61, unicodecsv
62, werkzeug
63, pytestCheckHook
64, freezegun
65, mkYarnPackage
66}:
67let
68 version = "2.2.4";
69
70 airflow-src = fetchFromGitHub rec {
71 owner = "apache";
72 repo = "airflow";
73 rev = version;
74 sha256 = "sha256-JCcEgCq1sB8lBaeJy7QQbWU00sGAh5vUmJAptF8M9qo=";
75 };
76
77 # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
78 # This replicates this shell script, fixing bugs in yarn.lock and package.json
79
80 airflow-frontend = mkYarnPackage {
81 name = "airflow-frontend";
82
83 src = "${airflow-src}/airflow/www";
84 packageJSON = ./package.json;
85 yarnLock = ./yarn.lock;
86 yarnNix = ./yarn.nix;
87
88 distPhase = "true";
89
90 configurePhase = ''
91 cp -r $node_modules node_modules
92 '';
93
94 buildPhase = ''
95 yarn --offline build
96 find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
97 '';
98
99 installPhase = ''
100 mkdir -p $out/static/
101 cp -r static/dist $out/static
102 '';
103 };
104
105in
106buildPythonPackage rec {
107 pname = "apache-airflow";
108 inherit version;
109 src = airflow-src;
110
111 disabled = pythonOlder "3.6";
112
113 propagatedBuildInputs = [
114 alembic
115 argcomplete
116 attrs
117 blinker
118 cached-property
119 cattrs
120 clickclick
121 colorlog
122 croniter
123 cryptography
124 dill
125 flask
126 flask-appbuilder
127 flask-caching
128 flask_login
129 flask_wtf
130 GitPython
131 graphviz
132 gunicorn
133 httpx
134 iso8601
135 importlib-resources
136 inflection
137 itsdangerous
138 jinja2
139 jsonschema
140 lazy-object-proxy
141 lockfile
142 markdown
143 markupsafe
144 marshmallow-oneofschema
145 numpy
146 openapi-spec-validator
147 pandas
148 pendulum
149 psutil
150 pygments
151 pyjwt
152 python-daemon
153 python-dateutil
154 python-nvd3
155 python-slugify
156 python3-openid
157 pyyaml
158 rich
159 setproctitle
160 sqlalchemy
161 sqlalchemy-jsonfield
162 swagger-ui-bundle
163 tabulate
164 tenacity
165 termcolor
166 unicodecsv
167 werkzeug
168 ] ++ lib.optionals (pythonOlder "3.7") [
169 dataclasses
170 ] ++ lib.optionals (pythonOlder "3.9") [
171 importlib-metadata
172 ];
173
174 buildInputs = [
175 airflow-frontend
176 ];
177
178 checkInputs = [
179 freezegun
180 pytestCheckHook
181 ];
182
183 INSTALL_PROVIDERS_FROM_SOURCES = "true";
184
185 postPatch = ''
186 substituteInPlace setup.cfg \
187 --replace "attrs>=20.0, <21.0" "attrs" \
188 --replace "cattrs~=1.1, <1.7.0" "cattrs" \
189 --replace "colorlog>=4.0.2, <6.0" "colorlog" \
190 --replace "croniter>=0.3.17, <1.1" "croniter" \
191 --replace "docutils<0.17" "docutils" \
192 --replace "flask-login>=0.3, <0.5" "flask-login" \
193 --replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \
194 --replace "flask>=1.1.0, <2.0" "flask" \
195 --replace "importlib_resources~=1.4" "importlib_resources" \
196 --replace "itsdangerous>=1.1.0, <2.0" "itsdangerous" \
197 --replace "markupsafe>=1.1.1, <2.0" "markupsafe" \
198 --replace "pyjwt<2" "pyjwt" \
199 --replace "python-slugify>=3.0.0,<5.0" "python-slugify" \
200 --replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \
201 --replace "tenacity~=6.2.0" "tenacity" \
202 --replace "werkzeug~=1.0, >=1.0.1" "werkzeug"
203
204 substituteInPlace tests/core/test_core.py \
205 --replace "/bin/bash" "${stdenv.shell}"
206 '' + lib.optionalString stdenv.isDarwin ''
207 # Fix failing test on Hydra
208 substituteInPlace airflow/utils/db.py \
209 --replace "/tmp/sqlite_default.db" "$TMPDIR/sqlite_default.db"
210 '';
211
212 # allow for gunicorn processes to have access to Python packages
213 makeWrapperArgs = [
214 "--prefix PYTHONPATH : $PYTHONPATH"
215 ];
216
217 preCheck = ''
218 export HOME=$(mktemp -d)
219 export AIRFLOW_HOME=$HOME
220 export AIRFLOW__CORE__UNIT_TEST_MODE=True
221 export AIRFLOW_DB="$HOME/airflow.db"
222 export PATH=$PATH:$out/bin
223
224 airflow version
225 airflow db init
226 airflow db reset -y
227 '';
228
229 pytestFlagsArray = [
230 "tests/core/test_core.py"
231 ];
232
233 disabledTests = lib.optionals stdenv.isDarwin [
234 "bash_operator_kill" # psutil.AccessDenied
235 ];
236
237 postInstall = ''
238 cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
239 '';
240
241 meta = with lib; {
242 description = "Programmatically author, schedule and monitor data pipelines";
243 homepage = "https://airflow.apache.org/";
244 license = licenses.asl20;
245 maintainers = with maintainers; [ bhipple costrouc ingenieroariel ];
246 # requires extremely outdated versions of multiple dependencies
247 broken = true;
248 };
249}