pyrox.dev
1{ lib
2, stdenv
3, python
4, buildPythonPackage
5, fetchFromGitHub
6, alembic
7, argcomplete
8, asgiref
9, attrs
10, blinker
11, cached-property
12, cattrs
13, clickclick
14, colorlog
15, configupdater
16, connexion
17, cron-descriptor
18, croniter
19, cryptography
20, deprecated
21, dill
22, flask
23, flask-login
24, flask-appbuilder
25, flask-caching
26, flask-session
27, flask-wtf
28, gitpython
29, google-re2
30, graphviz
31, gunicorn
32, httpx
33, iso8601
34, importlib-resources
35, importlib-metadata
36, inflection
37, itsdangerous
38, jinja2
39, jsonschema
40, lazy-object-proxy
41, linkify-it-py
42, lockfile
43, markdown
44, markupsafe
45, marshmallow-oneofschema
46, mdit-py-plugins
47, numpy
48, openapi-spec-validator
49, opentelemetry-api
50, opentelemetry-exporter-otlp
51, pandas
52, pathspec
53, pendulum
54, psutil
55, pydantic
56, pygments
57, pyjwt
58, python-daemon
59, python-dateutil
60, python-nvd3
61, python-slugify
62, python3-openid
63, pythonOlder
64, pythonRelaxDepsHook
65, pyyaml
66, rich
67, rich-argparse
68, setproctitle
69, sqlalchemy
70, sqlalchemy-jsonfield
71, swagger-ui-bundle
72, tabulate
73, tenacity
74, termcolor
75, typing-extensions
76, unicodecsv
77, werkzeug
78, freezegun
79, pytest-asyncio
80, pytestCheckHook
81, time-machine
82, mkYarnPackage
83, fetchYarnDeps
84, writeScript
85
86# Extra airflow providers to enable
87, enabledProviders ? []
88}:
89let
90 version = "2.7.3";
91
92 airflow-src = fetchFromGitHub rec {
93 owner = "apache";
94 repo = "airflow";
95 rev = "refs/tags/${version}";
96 # Download using the git protocol rather than using tarballs, because the
97 # GitHub archive tarballs don't appear to include tests
98 forceFetchGit = true;
99 hash = "sha256-+YbiKFZLigSDbHPaUKIl97kpezW1rIt/j09MMa6lwhQ=";
100 };
101
102 # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
103 # This replicates this shell script, fixing bugs in yarn.lock and package.json
104
105 airflow-frontend = mkYarnPackage rec {
106 name = "airflow-frontend";
107
108 src = "${airflow-src}/airflow/www";
109 packageJSON = ./package.json;
110
111 offlineCache = fetchYarnDeps {
112 yarnLock = "${src}/yarn.lock";
113 hash = "sha256-WQKuQgNp35fU6z7owequXOSwoUGJDJYcUgkjPDMOops=";
114 };
115
116 distPhase = "true";
117
118 # The webpack license plugin tries to create /licenses when given the
119 # original relative path
120 postPatch = ''
121 sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js
122 '';
123
124 configurePhase = ''
125 cp -r $node_modules node_modules
126 '';
127
128 buildPhase = ''
129 yarn --offline build
130 find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
131 '';
132
133 installPhase = ''
134 mkdir -p $out/static/
135 cp -r static/dist $out/static
136 '';
137 };
138
139 # Import generated file with metadata for provider dependencies and imports.
140 # Enable additional providers using enabledProviders above.
141 providers = import ./providers.nix;
142 getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps;
143 getProviderImports = provider: providers.${provider}.imports;
144 providerDependencies = lib.concatMap getProviderDeps enabledProviders;
145 providerImports = lib.concatMap getProviderImports enabledProviders;
146in
147buildPythonPackage rec {
148 pname = "apache-airflow";
149 inherit version;
150 src = airflow-src;
151
152 disabled = pythonOlder "3.7";
153
154 propagatedBuildInputs = [
155 alembic
156 argcomplete
157 asgiref
158 attrs
159 blinker
160 cached-property
161 cattrs
162 clickclick
163 colorlog
164 configupdater
165 connexion
166 cron-descriptor
167 croniter
168 cryptography
169 deprecated
170 dill
171 flask
172 flask-appbuilder
173 flask-caching
174 flask-session
175 flask-wtf
176 flask-login
177 gitpython
178 google-re2
179 graphviz
180 gunicorn
181 httpx
182 iso8601
183 importlib-resources
184 inflection
185 itsdangerous
186 jinja2
187 jsonschema
188 lazy-object-proxy
189 linkify-it-py
190 lockfile
191 markdown
192 markupsafe
193 marshmallow-oneofschema
194 mdit-py-plugins
195 numpy
196 openapi-spec-validator
197 opentelemetry-api
198 opentelemetry-exporter-otlp
199 pandas
200 pathspec
201 pendulum
202 psutil
203 pydantic
204 pygments
205 pyjwt
206 python-daemon
207 python-dateutil
208 python-nvd3
209 python-slugify
210 python3-openid
211 pyyaml
212 rich
213 rich-argparse
214 setproctitle
215 sqlalchemy
216 sqlalchemy-jsonfield
217 swagger-ui-bundle
218 tabulate
219 tenacity
220 termcolor
221 typing-extensions
222 unicodecsv
223 werkzeug
224 ] ++ lib.optionals (pythonOlder "3.9") [
225 importlib-metadata
226 ] ++ providerDependencies;
227
228 buildInputs = [
229 airflow-frontend
230 pythonRelaxDepsHook
231 ];
232
233 nativeCheckInputs = [
234 freezegun
235 pytest-asyncio
236 pytestCheckHook
237 time-machine
238 ];
239
240 # By default, source code of providers is included but unusable due to missing
241 # transitive dependencies. To enable a provider, add it to extraProviders
242 # above
243 INSTALL_PROVIDERS_FROM_SOURCES = "true";
244
245 postPatch = ''
246 # https://github.com/apache/airflow/issues/33854
247 substituteInPlace pyproject.toml \
248 --replace '[project]' $'[project]\nname = "apache-airflow"\nversion = "${version}"'
249 '' + lib.optionalString stdenv.isDarwin ''
250 # Fix failing test on Hydra
251 substituteInPlace airflow/utils/db.py \
252 --replace "/tmp/sqlite_default.db" "$TMPDIR/sqlite_default.db"
253 '';
254
255 pythonRelaxDeps = [
256 "colorlog"
257 "flask-appbuilder"
258 "opentelemetry-api"
259 "pathspec"
260 ];
261
262 # allow for gunicorn processes to have access to Python packages
263 makeWrapperArgs = [
264 "--prefix PYTHONPATH : $PYTHONPATH"
265 ];
266
267 postInstall = ''
268 cp -rv ${airflow-frontend}/static/dist $out/${python.sitePackages}/airflow/www/static
269 # Needed for pythonImportsCheck below
270 export HOME=$(mktemp -d)
271 '';
272
273 pythonImportsCheck = [
274 "airflow"
275 ] ++ providerImports;
276
277 preCheck = ''
278 export AIRFLOW_HOME=$HOME
279 export AIRFLOW__CORE__UNIT_TEST_MODE=True
280 export AIRFLOW_DB="$HOME/airflow.db"
281 export PATH=$PATH:$out/bin
282
283 airflow version
284 airflow db init
285 airflow db reset -y
286 '';
287
288 pytestFlagsArray = [
289 "tests/core/test_core.py"
290 ];
291
292 disabledTests = lib.optionals stdenv.isDarwin [
293 "bash_operator_kill" # psutil.AccessDenied
294 ];
295
296 # Updates yarn.lock and package.json
297 passthru.updateScript = writeScript "update.sh" ''
298 #!/usr/bin/env nix-shell
299 #!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix
300
301 set -euo pipefail
302
303 # Get new version
304 new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html |
305 pcregrep -o1 'Airflow ([0-9.]+).' | head -1)"
306 update-source-version ${pname} "$new_version"
307
308 # Update frontend
309 cd ./pkgs/servers/apache-airflow
310 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
311 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
312 yarn2nix > yarn.nix
313
314 # update provider dependencies
315 ./update-providers.py
316 '';
317
318 # Note on testing the web UI:
319 # You can (manually) test the web UI as follows:
320 #
321 # nix shell .#apache-airflow
322 # airflow db reset # WARNING: this will wipe any existing db state you might have!
323 # airflow db init
324 # airflow standalone
325 #
326 # Then navigate to the localhost URL using the credentials printed, try
327 # triggering the 'example_bash_operator' and 'example_bash_operator' DAGs and
328 # see if they report success.
329
330 meta = with lib; {
331 description = "Programmatically author, schedule and monitor data pipelines";
332 homepage = "https://airflow.apache.org/";
333 license = licenses.asl20;
334 maintainers = with maintainers; [ bhipple gbpdt ingenieroariel ];
335 knownVulnerabilities = [
336 "CVE-2023-50943"
337 "CVE-2023-50944"
338 ];
339 };
340}