tjh.dev
1{ lib
2, stdenv
3, python
4, buildPythonPackage
5, fetchFromGitHub
6, alembic
7, argcomplete
8, asgiref
9, attrs
10, blinker
11, cached-property
12, cattrs
13, clickclick
14, colorlog
15, configupdater
16, connexion
17, cron-descriptor
18, croniter
19, cryptography
20, deprecated
21, dill
22, flask
23, flask-login
24, flask-appbuilder
25, flask-caching
26, flask-session
27, flask-wtf
28, gitpython
29, graphviz
30, gunicorn
31, httpx
32, iso8601
33, importlib-resources
34, importlib-metadata
35, inflection
36, itsdangerous
37, jinja2
38, jsonschema
39, lazy-object-proxy
40, linkify-it-py
41, lockfile
42, markdown
43, markupsafe
44, marshmallow-oneofschema
45, mdit-py-plugins
46, numpy
47, openapi-spec-validator
48, pandas
49, pathspec
50, pendulum
51, psutil
52, pydantic
53, pygments
54, pyjwt
55, python-daemon
56, python-dateutil
57, python-nvd3
58, python-slugify
59, python3-openid
60, pythonOlder
61, pyyaml
62, rich
63, rich-argparse
64, setproctitle
65, sqlalchemy
66, sqlalchemy-jsonfield
67, swagger-ui-bundle
68, tabulate
69, tenacity
70, termcolor
71, typing-extensions
72, unicodecsv
73, werkzeug
74, freezegun
75, pytest-asyncio
76, pytestCheckHook
77, time-machine
78, mkYarnPackage
79, writeScript
80
81# Extra airflow providers to enable
82, enabledProviders ? []
83}:
84let
85 version = "2.6.0";
86
87 airflow-src = fetchFromGitHub rec {
88 owner = "apache";
89 repo = "airflow";
90 rev = "refs/tags/${version}";
91 # Download using the git protocol rather than using tarballs, because the
92 # GitHub archive tarballs don't appear to include tests
93 forceFetchGit = true;
94 hash = "sha256-CsLOj68+tSOI7drZq6QH+C2EB/9trg5cFAYAQp/06m8=";
95 };
96
97 # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
98 # This replicates this shell script, fixing bugs in yarn.lock and package.json
99
100 airflow-frontend = mkYarnPackage {
101 name = "airflow-frontend";
102
103 src = "${airflow-src}/airflow/www";
104 packageJSON = ./package.json;
105 yarnLock = ./yarn.lock;
106 yarnNix = ./yarn.nix;
107
108 distPhase = "true";
109
110 # The webpack license plugin tries to create /licenses when given the
111 # original relative path
112 postPatch = ''
113 sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js
114 '';
115
116 configurePhase = ''
117 cp -r $node_modules node_modules
118 '';
119
120 buildPhase = ''
121 yarn --offline build
122 find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
123 '';
124
125 installPhase = ''
126 mkdir -p $out/static/
127 cp -r static/dist $out/static
128 '';
129 };
130
131 # Import generated file with metadata for provider dependencies and imports.
132 # Enable additional providers using enabledProviders above.
133 providers = import ./providers.nix;
134 getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps;
135 getProviderImports = provider: providers.${provider}.imports;
136 providerDependencies = lib.concatMap getProviderDeps enabledProviders;
137 providerImports = lib.concatMap getProviderImports enabledProviders;
138in
139buildPythonPackage rec {
140 pname = "apache-airflow";
141 inherit version;
142 src = airflow-src;
143
144 disabled = pythonOlder "3.7";
145
146 propagatedBuildInputs = [
147 alembic
148 argcomplete
149 asgiref
150 attrs
151 blinker
152 cached-property
153 cattrs
154 clickclick
155 colorlog
156 configupdater
157 connexion
158 cron-descriptor
159 croniter
160 cryptography
161 deprecated
162 dill
163 flask
164 flask-appbuilder
165 flask-caching
166 flask-session
167 flask-wtf
168 flask-login
169 gitpython
170 graphviz
171 gunicorn
172 httpx
173 iso8601
174 importlib-resources
175 inflection
176 itsdangerous
177 jinja2
178 jsonschema
179 lazy-object-proxy
180 linkify-it-py
181 lockfile
182 markdown
183 markupsafe
184 marshmallow-oneofschema
185 mdit-py-plugins
186 numpy
187 openapi-spec-validator
188 pandas
189 pathspec
190 pendulum
191 psutil
192 pydantic
193 pygments
194 pyjwt
195 python-daemon
196 python-dateutil
197 python-nvd3
198 python-slugify
199 python3-openid
200 pyyaml
201 rich
202 rich-argparse
203 setproctitle
204 sqlalchemy
205 sqlalchemy-jsonfield
206 swagger-ui-bundle
207 tabulate
208 tenacity
209 termcolor
210 typing-extensions
211 unicodecsv
212 werkzeug
213 ] ++ lib.optionals (pythonOlder "3.9") [
214 importlib-metadata
215 ] ++ providerDependencies;
216
217 buildInputs = [
218 airflow-frontend
219 ];
220
221 nativeCheckInputs = [
222 freezegun
223 pytest-asyncio
224 pytestCheckHook
225 time-machine
226 ];
227
228 # By default, source code of providers is included but unusable due to missing
229 # transitive dependencies. To enable a provider, add it to extraProviders
230 # above
231 INSTALL_PROVIDERS_FROM_SOURCES = "true";
232
233 postPatch = ''
234 substituteInPlace setup.cfg \
235 --replace "colorlog>=4.0.2, <5.0" "colorlog" \
236 --replace "flask-appbuilder==4.3.0" "flask-appbuilder>=4.3.0" \
237 --replace "pathspec~=0.9.0" "pathspec"
238 '' + lib.optionalString stdenv.isDarwin ''
239 # Fix failing test on Hydra
240 substituteInPlace airflow/utils/db.py \
241 --replace "/tmp/sqlite_default.db" "$TMPDIR/sqlite_default.db"
242 '';
243
244 # allow for gunicorn processes to have access to Python packages
245 makeWrapperArgs = [
246 "--prefix PYTHONPATH : $PYTHONPATH"
247 ];
248
249 postInstall = ''
250 cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
251 # Needed for pythonImportsCheck below
252 export HOME=$(mktemp -d)
253 '';
254
255 pythonImportsCheck = [
256 "airflow"
257 ] ++ providerImports;
258
259 preCheck = ''
260 export AIRFLOW_HOME=$HOME
261 export AIRFLOW__CORE__UNIT_TEST_MODE=True
262 export AIRFLOW_DB="$HOME/airflow.db"
263 export PATH=$PATH:$out/bin
264
265 airflow version
266 airflow db init
267 airflow db reset -y
268 '';
269
270 pytestFlagsArray = [
271 "tests/core/test_core.py"
272 ];
273
274 disabledTests = lib.optionals stdenv.isDarwin [
275 "bash_operator_kill" # psutil.AccessDenied
276 ];
277
278 # Updates yarn.lock and package.json
279 passthru.updateScript = writeScript "update.sh" ''
280 #!/usr/bin/env nix-shell
281 #!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix
282
283 set -euo pipefail
284
285 # Get new version
286 new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html |
287 pcregrep -o1 'Airflow ([0-9.]+).' | head -1)"
288 update-source-version ${pname} "$new_version"
289
290 # Update frontend
291 cd ./pkgs/development/python-modules/apache-airflow
292 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
293 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
294 yarn2nix > yarn.nix
295
296 # update provider dependencies
297 ./update-providers.py
298 '';
299
300 # Note on testing the web UI:
301 # You can (manually) test the web UI as follows:
302 #
303 # nix shell .#apache-airflow
304 # airflow db reset # WARNING: this will wipe any existing db state you might have!
305 # airflow db init
306 # airflow standalone
307 #
308 # Then navigate to the localhost URL using the credentials printed, try
309 # triggering the 'example_bash_operator' and 'example_bash_operator' DAGs and
310 # see if they report success.
311
312 meta = with lib; {
313 description = "Programmatically author, schedule and monitor data pipelines";
314 homepage = "https://airflow.apache.org/";
315 license = licenses.asl20;
316 maintainers = with maintainers; [ bhipple gbpdt ingenieroariel ];
317 };
318}