tjh.dev
1{
2 lib,
3 stdenv,
4 python,
5 buildPythonPackage,
6 fetchFromGitHub,
7 alembic,
8 argcomplete,
9 asgiref,
10 attrs,
11 blinker,
12 cached-property,
13 cattrs,
14 clickclick,
15 colorlog,
16 configupdater,
17 connexion,
18 cron-descriptor,
19 croniter,
20 cryptography,
21 deprecated,
22 dill,
23 flask,
24 flask-login,
25 flask-appbuilder,
26 flask-caching,
27 flask-session,
28 flask-wtf,
29 gitpython,
30 google-re2,
31 graphviz,
32 gunicorn,
33 httpx,
34 iso8601,
35 importlib-resources,
36 importlib-metadata,
37 inflection,
38 itsdangerous,
39 jinja2,
40 jsonschema,
41 lazy-object-proxy,
42 linkify-it-py,
43 lockfile,
44 markdown,
45 markupsafe,
46 marshmallow-oneofschema,
47 mdit-py-plugins,
48 numpy,
49 openapi-spec-validator,
50 opentelemetry-api,
51 opentelemetry-exporter-otlp,
52 pandas,
53 pathspec,
54 pendulum,
55 psutil,
56 pydantic,
57 pygments,
58 pyjwt,
59 python-daemon,
60 python-dateutil,
61 python-nvd3,
62 python-slugify,
63 python3-openid,
64 pythonOlder,
65 pyyaml,
66 rich,
67 rich-argparse,
68 setproctitle,
69 sqlalchemy,
70 sqlalchemy-jsonfield,
71 swagger-ui-bundle,
72 tabulate,
73 tenacity,
74 termcolor,
75 typing-extensions,
76 unicodecsv,
77 werkzeug,
78 freezegun,
79 pytest-asyncio,
80 pytestCheckHook,
81 time-machine,
82 mkYarnPackage,
83 fetchYarnDeps,
84 writeScript,
85
86 # Extra airflow providers to enable
87 enabledProviders ? [ ],
88}:
89let
90 version = "2.7.3";
91
92 airflow-src = fetchFromGitHub {
93 owner = "apache";
94 repo = "airflow";
95 rev = "refs/tags/${version}";
96 # Download using the git protocol rather than using tarballs, because the
97 # GitHub archive tarballs don't appear to include tests
98 forceFetchGit = true;
99 hash = "sha256-+YbiKFZLigSDbHPaUKIl97kpezW1rIt/j09MMa6lwhQ=";
100 };
101
102 # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
103 # This replicates this shell script, fixing bugs in yarn.lock and package.json
104
105 airflow-frontend = mkYarnPackage rec {
106 name = "airflow-frontend";
107
108 src = "${airflow-src}/airflow/www";
109 packageJSON = ./package.json;
110
111 offlineCache = fetchYarnDeps {
112 yarnLock = "${src}/yarn.lock";
113 hash = "sha256-WQKuQgNp35fU6z7owequXOSwoUGJDJYcUgkjPDMOops=";
114 };
115
116 distPhase = "true";
117
118 # The webpack license plugin tries to create /licenses when given the
119 # original relative path
120 postPatch = ''
121 sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js
122 '';
123
124 configurePhase = ''
125 cp -r $node_modules node_modules
126 '';
127
128 buildPhase = ''
129 yarn --offline build
130 find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
131 '';
132
133 installPhase = ''
134 mkdir -p $out/static/
135 cp -r static/dist $out/static
136 '';
137 };
138
139 # Import generated file with metadata for provider dependencies and imports.
140 # Enable additional providers using enabledProviders above.
141 providers = import ./providers.nix;
142 getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps;
143 getProviderImports = provider: providers.${provider}.imports;
144 providerDependencies = lib.concatMap getProviderDeps enabledProviders;
145 providerImports = lib.concatMap getProviderImports enabledProviders;
146in
147buildPythonPackage rec {
148 pname = "apache-airflow";
149 inherit version;
150 format = "setuptools";
151 src = airflow-src;
152
153 disabled = pythonOlder "3.7";
154
155 propagatedBuildInputs = [
156 alembic
157 argcomplete
158 asgiref
159 attrs
160 blinker
161 cached-property
162 cattrs
163 clickclick
164 colorlog
165 configupdater
166 connexion
167 cron-descriptor
168 croniter
169 cryptography
170 deprecated
171 dill
172 flask
173 flask-appbuilder
174 flask-caching
175 flask-session
176 flask-wtf
177 flask-login
178 gitpython
179 google-re2
180 graphviz
181 gunicorn
182 httpx
183 iso8601
184 importlib-resources
185 inflection
186 itsdangerous
187 jinja2
188 jsonschema
189 lazy-object-proxy
190 linkify-it-py
191 lockfile
192 markdown
193 markupsafe
194 marshmallow-oneofschema
195 mdit-py-plugins
196 numpy
197 openapi-spec-validator
198 opentelemetry-api
199 opentelemetry-exporter-otlp
200 pandas
201 pathspec
202 pendulum
203 psutil
204 pydantic
205 pygments
206 pyjwt
207 python-daemon
208 python-dateutil
209 python-nvd3
210 python-slugify
211 python3-openid
212 pyyaml
213 rich
214 rich-argparse
215 setproctitle
216 sqlalchemy
217 sqlalchemy-jsonfield
218 swagger-ui-bundle
219 tabulate
220 tenacity
221 termcolor
222 typing-extensions
223 unicodecsv
224 werkzeug
225 ]
226 ++ lib.optionals (pythonOlder "3.9") [
227 importlib-metadata
228 ]
229 ++ providerDependencies;
230
231 buildInputs = [
232 airflow-frontend
233 ];
234
235 nativeCheckInputs = [
236 freezegun
237 pytest-asyncio
238 pytestCheckHook
239 time-machine
240 ];
241
242 # By default, source code of providers is included but unusable due to missing
243 # transitive dependencies. To enable a provider, add it to extraProviders
244 # above
245 INSTALL_PROVIDERS_FROM_SOURCES = "true";
246
247 postPatch = ''
248 # https://github.com/apache/airflow/issues/33854
249 substituteInPlace pyproject.toml \
250 --replace '[project]' $'[project]\nname = "apache-airflow"\nversion = "${version}"'
251 ''
252 + lib.optionalString stdenv.hostPlatform.isDarwin ''
253 # Fix failing test on Hydra
254 substituteInPlace airflow/utils/db.py \
255 --replace "/tmp/sqlite_default.db" "$TMPDIR/sqlite_default.db"
256 '';
257
258 pythonRelaxDeps = [
259 "colorlog"
260 "flask-appbuilder"
261 "opentelemetry-api"
262 "pathspec"
263 ];
264
265 # allow for gunicorn processes to have access to Python packages
266 makeWrapperArgs = [
267 "--prefix PYTHONPATH : $PYTHONPATH"
268 ];
269
270 postInstall = ''
271 cp -rv ${airflow-frontend}/static/dist $out/${python.sitePackages}/airflow/www/static
272 # Needed for pythonImportsCheck below
273 export HOME=$(mktemp -d)
274 '';
275
276 pythonImportsCheck = [
277 "airflow"
278 ]
279 ++ providerImports;
280
281 preCheck = ''
282 export AIRFLOW_HOME=$HOME
283 export AIRFLOW__CORE__UNIT_TEST_MODE=True
284 export AIRFLOW_DB="$HOME/airflow.db"
285 export PATH=$PATH:$out/bin
286
287 airflow version
288 airflow db init
289 airflow db reset -y
290 '';
291
292 enabledTestPaths = [
293 "tests/core/test_core.py"
294 ];
295
296 disabledTests = lib.optionals stdenv.hostPlatform.isDarwin [
297 "bash_operator_kill" # psutil.AccessDenied
298 ];
299
300 # Updates yarn.lock and package.json
301 passthru.updateScript = writeScript "update.sh" ''
302 #!/usr/bin/env nix-shell
303 #!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix
304
305 set -euo pipefail
306
307 # Get new version
308 new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html |
309 pcregrep -o1 'Airflow ([0-9.]+).' | head -1)"
310 update-source-version ${pname} "$new_version"
311
312 # Update frontend
313 cd ./pkgs/servers/apache-airflow
314 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
315 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
316 yarn2nix > yarn.nix
317
318 # update provider dependencies
319 ./update-providers.py
320 '';
321
322 # Note on testing the web UI:
323 # You can (manually) test the web UI as follows:
324 #
325 # nix shell .#apache-airflow
326 # airflow db reset # WARNING: this will wipe any existing db state you might have!
327 # airflow db init
328 # airflow standalone
329 #
330 # Then navigate to the localhost URL using the credentials printed, try
331 # triggering the 'example_bash_operator' and 'example_bash_operator' DAGs and
332 # see if they report success.
333
334 meta = with lib; {
335 description = "Programmatically author, schedule and monitor data pipelines";
336 homepage = "https://airflow.apache.org/";
337 license = licenses.asl20;
338 maintainers = with maintainers; [
339 bhipple
340 gbpdt
341 ingenieroariel
342 ];
343 knownVulnerabilities = [
344 "CVE-2023-50943"
345 "CVE-2023-50944"
346 ];
347 };
348}