tjh.dev
1{ lib
2, stdenv
3, python
4, buildPythonPackage
5, fetchFromGitHub
6, alembic
7, argcomplete
8, attrs
9, blinker
10, cached-property
11, cattrs
12, clickclick
13, colorlog
14, configupdater
15, connexion
16, cron-descriptor
17, croniter
18, cryptography
19, deprecated
20, dill
21, flask
22, flask-login
23, flask-appbuilder
24, flask-caching
25, flask-session
26, flask-wtf
27, gitpython
28, graphviz
29, gunicorn
30, httpx
31, iso8601
32, importlib-resources
33, importlib-metadata
34, inflection
35, itsdangerous
36, jinja2
37, jsonschema
38, lazy-object-proxy
39, linkify-it-py
40, lockfile
41, markdown
42, markupsafe
43, marshmallow-oneofschema
44, mdit-py-plugins
45, numpy
46, openapi-spec-validator
47, pandas
48, pathspec
49, pendulum
50, psutil
51, pygments
52, pyjwt
53, python-daemon
54, python-dateutil
55, python-nvd3
56, python-slugify
57, python3-openid
58, pythonOlder
59, pyyaml
60, rich
61, setproctitle
62, sqlalchemy
63, sqlalchemy-jsonfield
64, swagger-ui-bundle
65, tabulate
66, tenacity
67, termcolor
68, typing-extensions
69, unicodecsv
70, werkzeug
71, pytestCheckHook
72, freezegun
73, mkYarnPackage
74, writeScript
75
76# Extra airflow providers to enable
77, enabledProviders ? []
78}:
79let
80 version = "2.5.1";
81
82 airflow-src = fetchFromGitHub rec {
83 owner = "apache";
84 repo = "airflow";
85 rev = "refs/tags/${version}";
86 # Download using the git protocol rather than using tarballs, because the
87 # GitHub archive tarballs don't appear to include tests
88 forceFetchGit = true;
89 hash = "sha256-BuJfE6SONTNonUvacOAIdZe0QicdBtx7k186TJZpQOs=";
90 };
91
92 # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
93 # This replicates this shell script, fixing bugs in yarn.lock and package.json
94
95 airflow-frontend = mkYarnPackage {
96 name = "airflow-frontend";
97
98 src = "${airflow-src}/airflow/www";
99 packageJSON = ./package.json;
100 yarnLock = ./yarn.lock;
101 yarnNix = ./yarn.nix;
102
103 distPhase = "true";
104
105 # The webpack license plugin tries to create /licenses when given the
106 # original relative path
107 postPatch = ''
108 sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js
109 '';
110
111 configurePhase = ''
112 cp -r $node_modules node_modules
113 '';
114
115 buildPhase = ''
116 yarn --offline build
117 find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
118 '';
119
120 installPhase = ''
121 mkdir -p $out/static/
122 cp -r static/dist $out/static
123 '';
124 };
125
126 # Import generated file with metadata for provider dependencies and imports.
127 # Enable additional providers using enabledProviders above.
128 providers = import ./providers.nix;
129 getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps;
130 getProviderImports = provider: providers.${provider}.imports;
131 providerDependencies = lib.concatMap getProviderDeps enabledProviders;
132 providerImports = lib.concatMap getProviderImports enabledProviders;
133in
134buildPythonPackage rec {
135 pname = "apache-airflow";
136 inherit version;
137 src = airflow-src;
138
139 disabled = pythonOlder "3.7";
140
141 propagatedBuildInputs = [
142 alembic
143 argcomplete
144 attrs
145 blinker
146 cached-property
147 cattrs
148 clickclick
149 colorlog
150 configupdater
151 connexion
152 cron-descriptor
153 croniter
154 cryptography
155 deprecated
156 dill
157 flask
158 flask-appbuilder
159 flask-caching
160 flask-session
161 flask-wtf
162 flask-login
163 gitpython
164 graphviz
165 gunicorn
166 httpx
167 iso8601
168 importlib-resources
169 inflection
170 itsdangerous
171 jinja2
172 jsonschema
173 lazy-object-proxy
174 linkify-it-py
175 lockfile
176 markdown
177 markupsafe
178 marshmallow-oneofschema
179 mdit-py-plugins
180 numpy
181 openapi-spec-validator
182 pandas
183 pathspec
184 pendulum
185 psutil
186 pygments
187 pyjwt
188 python-daemon
189 python-dateutil
190 python-nvd3
191 python-slugify
192 python3-openid
193 pyyaml
194 rich
195 setproctitle
196 sqlalchemy
197 sqlalchemy-jsonfield
198 swagger-ui-bundle
199 tabulate
200 tenacity
201 termcolor
202 typing-extensions
203 unicodecsv
204 werkzeug
205 ] ++ lib.optionals (pythonOlder "3.9") [
206 importlib-metadata
207 ] ++ providerDependencies;
208
209 buildInputs = [
210 airflow-frontend
211 ];
212
213 nativeCheckInputs = [
214 freezegun
215 pytestCheckHook
216 ];
217
218 # By default, source code of providers is included but unusable due to missing
219 # transitive dependencies. To enable a provider, add it to extraProviders
220 # above
221 INSTALL_PROVIDERS_FROM_SOURCES = "true";
222
223 postPatch = ''
224 substituteInPlace setup.cfg \
225 --replace "colorlog>=4.0.2, <5.0" "colorlog" \
226 --replace "flask-appbuilder==4.1.4" "flask-appbuilder>=4.1.3" \
227 --replace "pathspec~=0.9.0" "pathspec"
228 '' + lib.optionalString stdenv.isDarwin ''
229 # Fix failing test on Hydra
230 substituteInPlace airflow/utils/db.py \
231 --replace "/tmp/sqlite_default.db" "$TMPDIR/sqlite_default.db"
232 '';
233
234 # allow for gunicorn processes to have access to Python packages
235 makeWrapperArgs = [
236 "--prefix PYTHONPATH : $PYTHONPATH"
237 ];
238
239 postInstall = ''
240 cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
241 # Needed for pythonImportsCheck below
242 export HOME=$(mktemp -d)
243 '';
244
245 pythonImportsCheck = [
246 "airflow"
247 ] ++ providerImports;
248
249 preCheck = ''
250 export AIRFLOW_HOME=$HOME
251 export AIRFLOW__CORE__UNIT_TEST_MODE=True
252 export AIRFLOW_DB="$HOME/airflow.db"
253 export PATH=$PATH:$out/bin
254
255 airflow version
256 airflow db init
257 airflow db reset -y
258 '';
259
260 pytestFlagsArray = [
261 "tests/core/test_core.py"
262 ];
263
264 disabledTests = lib.optionals stdenv.isDarwin [
265 "bash_operator_kill" # psutil.AccessDenied
266 ];
267
268 # Updates yarn.lock and package.json
269 passthru.updateScript = writeScript "update.sh" ''
270 #!/usr/bin/env nix-shell
271 #!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix
272
273 set -euo pipefail
274
275 # Get new version
276 new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html |
277 pcregrep -o1 'Airflow ([0-9.]+).' | head -1)"
278 update-source-version ${pname} "$new_version"
279
280 # Update frontend
281 cd ./pkgs/development/python-modules/apache-airflow
282 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
283 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
284 # Revert this commit which seems to break with our version of yarn
285 # https://github.com/apache/airflow/commit/b9e133e40c2848b0d555051a99bf8d2816fd28a7
286 patch -p3 < 0001-Revert-fix-yarn-warning-from-d3-color-27139.patch
287 yarn2nix > yarn.nix
288
289 # update provider dependencies
290 ./update-providers.py
291 '';
292
293 # Note on testing the web UI:
294 # You can (manually) test the web UI as follows:
295 #
296 # nix shell .#python3Packages.apache-airflow
297 # airflow db reset # WARNING: this will wipe any existing db state you might have!
298 # airflow db init
299 # airflow standalone
300 #
301 # Then navigate to the localhost URL using the credentials printed, try
302 # triggering the 'example_bash_operator' and 'example_bash_operator' DAGs and
303 # see if they report success.
304
305 meta = with lib; {
306 description = "Programmatically author, schedule and monitor data pipelines";
307 homepage = "https://airflow.apache.org/";
308 license = licenses.asl20;
309 maintainers = with maintainers; [ bhipple gbpdt ingenieroariel ];
310 };
311}