pyrox.dev
1{ lib
2, stdenv
3, python
4, buildPythonPackage
5, fetchFromGitHub
6, alembic
7, argcomplete
8, attrs
9, blinker
10, cached-property
11, cattrs
12, clickclick
13, colorlog
14, croniter
15, cryptography
16, dill
17, flask
18, flask-appbuilder
19, flask-caching
20, flask_login
21, flask_wtf
22, GitPython
23, graphviz
24, gunicorn
25, httpx
26, iso8601
27, importlib-resources
28, importlib-metadata
29, inflection
30, itsdangerous
31, jinja2
32, jsonschema
33, lazy-object-proxy
34, lockfile
35, markdown
36, markupsafe
37, marshmallow-oneofschema
38, numpy
39, openapi-spec-validator
40, pandas
41, pendulum
42, psutil
43, pygments
44, pyjwt
45, python-daemon
46, python-dateutil
47, python-nvd3
48, python-slugify
49, python3-openid
50, pyyaml
51, rich
52, setproctitle
53, sqlalchemy
54, sqlalchemy-jsonfield
55, swagger-ui-bundle
56, tabulate
57, tenacity
58, termcolor
59, unicodecsv
60, werkzeug
61, pytestCheckHook
62, freezegun
63, mkYarnPackage
64}:
65let
66
67 version = "2.1.4";
68
69 airflow-src = fetchFromGitHub rec {
70 owner = "apache";
71 repo = "airflow";
72 rev = version;
73 sha256 = "12nxjaz4afkq30s42x3rbsci8jiw2k5zjngsc8i190fasbacbnbs";
74 };
75
76 # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
77 # This replicates this shell script, fixing bugs in yarn.lock and package.json
78
79 airflow-frontend = mkYarnPackage {
80 name = "airflow-frontend";
81
82 src = "${airflow-src}/airflow/www";
83 packageJSON = ./package.json;
84 yarnLock = ./yarn.lock;
85 yarnNix = ./yarn.nix;
86
87 distPhase = "true";
88
89 configurePhase = ''
90 cp -r $node_modules node_modules
91 '';
92
93 buildPhase = ''
94 yarn --offline build
95 find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
96 '';
97
98 installPhase = ''
99 mkdir -p $out/static/
100 cp -r static/dist $out/static
101 '';
102 };
103
104in
105buildPythonPackage rec {
106 pname = "apache-airflow";
107 inherit version;
108 src = airflow-src;
109
110 propagatedBuildInputs = [
111 alembic
112 argcomplete
113 attrs
114 blinker
115 cached-property
116 cattrs
117 clickclick
118 colorlog
119 croniter
120 cryptography
121 dill
122 flask
123 flask-appbuilder
124 flask-caching
125 flask_login
126 flask_wtf
127 GitPython
128 graphviz
129 gunicorn
130 httpx
131 iso8601
132 importlib-resources
133 importlib-metadata
134 inflection
135 itsdangerous
136 jinja2
137 jsonschema
138 lazy-object-proxy
139 lockfile
140 markdown
141 markupsafe
142 marshmallow-oneofschema
143 numpy
144 openapi-spec-validator
145 pandas
146 pendulum
147 psutil
148 pygments
149 pyjwt
150 python-daemon
151 python-dateutil
152 python-nvd3
153 python-slugify
154 python3-openid
155 pyyaml
156 rich
157 setproctitle
158 sqlalchemy
159 sqlalchemy-jsonfield
160 swagger-ui-bundle
161 tabulate
162 tenacity
163 termcolor
164 unicodecsv
165 werkzeug
166 ];
167
168 buildInputs = [
169 airflow-frontend
170 ];
171
172 checkInputs = [
173 freezegun
174 pytestCheckHook
175 ];
176
177 INSTALL_PROVIDERS_FROM_SOURCES = "true";
178
179 postPatch = ''
180 substituteInPlace setup.cfg \
181 --replace "importlib_resources~=1.4" "importlib_resources" \
182 --replace "importlib_metadata~=1.7" "importlib_metadata" \
183 --replace "tenacity~=6.2.0" "tenacity" \
184 --replace "pyjwt<2" "pyjwt" \
185 --replace "flask>=1.1.0, <2.0" "flask" \
186 --replace "flask-login>=0.3, <0.5" "flask-login" \
187 --replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \
188 --replace "jinja2>=2.10.1, <2.12.0" "jinja2" \
189 --replace "attrs>=20.0, <21.0" "attrs" \
190 --replace "cattrs~=1.1, <1.7.0" "cattrs" \
191 --replace "markupsafe>=1.1.1, <2.0" "markupsafe" \
192 --replace "docutils<0.17" "docutils" \
193 --replace "sqlalchemy>=1.3.18, <1.4" "sqlalchemy" \
194 --replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \
195 --replace "werkzeug~=1.0, >=1.0.1" "werkzeug" \
196 --replace "itsdangerous>=1.1.0, <2.0" "itsdangerous" \
197 --replace "python-slugify>=3.0.0,<5.0" "python-slugify" \
198 --replace "colorlog>=4.0.2, <6.0" "colorlog"
199
200 substituteInPlace tests/core/test_core.py \
201 --replace "/bin/bash" "${stdenv.shell}"
202 '' + lib.optionalString stdenv.isDarwin ''
203 # Fix failing test on Hydra
204 substituteInPlace airflow/utils/db.py \
205 --replace "/tmp/sqlite_default.db" "$TMPDIR/sqlite_default.db"
206 '';
207
208 # allow for gunicorn processes to have access to python packages
209 makeWrapperArgs = [ "--prefix PYTHONPATH : $PYTHONPATH" ];
210
211 preCheck = ''
212 export HOME=$(mktemp -d)
213 export AIRFLOW_HOME=$HOME
214 export AIRFLOW__CORE__UNIT_TEST_MODE=True
215 export AIRFLOW_DB="$HOME/airflow.db"
216 export PATH=$PATH:$out/bin
217
218 airflow version
219 airflow db init
220 airflow db reset -y
221 '';
222
223 pytestFlagsArray = [
224 "tests/core/test_core.py"
225 ];
226
227 disabledTests = lib.optionals stdenv.isDarwin [
228 "bash_operator_kill" # psutil.AccessDenied
229 ];
230
231 postInstall = ''
232 cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
233 '';
234
235 meta = with lib; {
236 description = "Programmatically author, schedule and monitor data pipelines";
237 homepage = "http://airflow.apache.org/";
238 license = licenses.asl20;
239 maintainers = with maintainers; [ bhipple costrouc ingenieroariel ];
240 };
241}