pyrox.dev
1{ lib
2, stdenv
3, python
4, buildPythonPackage
5, fetchFromGitHub
6, alembic
7, argcomplete
8, attrs
9, blinker
10, cached-property
11, cattrs
12, clickclick
13, colorlog
14, configupdater
15, connexion
16, cron-descriptor
17, croniter
18, cryptography
19, dataclasses
20, deprecated
21, dill
22, flask
23, flask-login
24, flask-appbuilder
25, flask-caching
26, flask-session
27, flask-wtf
28, GitPython
29, graphviz
30, gunicorn
31, httpx
32, iso8601
33, importlib-resources
34, importlib-metadata
35, inflection
36, itsdangerous
37, jinja2
38, jsonschema
39, lazy-object-proxy
40, linkify-it-py
41, lockfile
42, markdown
43, markupsafe
44, marshmallow-oneofschema
45, mdit-py-plugins
46, numpy
47, openapi-spec-validator
48, pandas
49, pathspec
50, pendulum
51, psutil
52, pygments
53, pyjwt
54, python-daemon
55, python-dateutil
56, python-nvd3
57, python-slugify
58, python3-openid
59, pythonOlder
60, pyyaml
61, rich
62, setproctitle
63, sqlalchemy
64, sqlalchemy-jsonfield
65, swagger-ui-bundle
66, tabulate
67, tenacity
68, termcolor
69, typing-extensions
70, unicodecsv
71, werkzeug
72, pytestCheckHook
73, freezegun
74, mkYarnPackage
75, writeScript
76
77# Extra airflow providers to enable
78, enabledProviders ? []
79}:
80let
81 version = "2.4.1";
82
83 airflow-src = fetchFromGitHub rec {
84 owner = "apache";
85 repo = "airflow";
86 rev = "refs/tags/${version}";
87 # Required because the GitHub archive tarballs don't appear to include tests
88 leaveDotGit = true;
89 sha256 = "sha256-HpPL/ocV7hRhYXsjfXMYvlP83Vh15kXyjBgubsaqaE8=";
90 };
91
92 # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
93 # This replicates this shell script, fixing bugs in yarn.lock and package.json
94
95 airflow-frontend = mkYarnPackage {
96 name = "airflow-frontend";
97
98 src = "${airflow-src}/airflow/www";
99 packageJSON = ./package.json;
100 yarnLock = ./yarn.lock;
101 yarnNix = ./yarn.nix;
102
103 distPhase = "true";
104
105 # The webpack license plugin tries to create /licenses when given the
106 # original relative path
107 postPatch = ''
108 sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js
109 '';
110
111 configurePhase = ''
112 cp -r $node_modules node_modules
113 '';
114
115 buildPhase = ''
116 yarn --offline build
117 find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
118 '';
119
120 installPhase = ''
121 mkdir -p $out/static/
122 cp -r static/dist $out/static
123 '';
124 };
125
126 # Import generated file with metadata for provider dependencies and imports.
127 # Enable additional providers using enabledProviders above.
128 providers = import ./providers.nix;
129 getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps;
130 getProviderImports = provider: providers.${provider}.imports;
131 providerDependencies = lib.concatMap getProviderDeps enabledProviders;
132 providerImports = lib.concatMap getProviderImports enabledProviders;
133in
134buildPythonPackage rec {
135 pname = "apache-airflow";
136 inherit version;
137 src = airflow-src;
138
139 disabled = pythonOlder "3.6";
140
141 propagatedBuildInputs = [
142 alembic
143 argcomplete
144 attrs
145 blinker
146 cached-property
147 cattrs
148 clickclick
149 colorlog
150 configupdater
151 connexion
152 cron-descriptor
153 croniter
154 cryptography
155 deprecated
156 dill
157 flask
158 flask-appbuilder
159 flask-caching
160 flask-session
161 flask-wtf
162 flask-login
163 GitPython
164 graphviz
165 gunicorn
166 httpx
167 iso8601
168 importlib-resources
169 inflection
170 itsdangerous
171 jinja2
172 jsonschema
173 lazy-object-proxy
174 linkify-it-py
175 lockfile
176 markdown
177 markupsafe
178 marshmallow-oneofschema
179 mdit-py-plugins
180 numpy
181 openapi-spec-validator
182 pandas
183 pathspec
184 pendulum
185 psutil
186 pygments
187 pyjwt
188 python-daemon
189 python-dateutil
190 python-nvd3
191 python-slugify
192 python3-openid
193 pyyaml
194 rich
195 setproctitle
196 sqlalchemy
197 sqlalchemy-jsonfield
198 swagger-ui-bundle
199 tabulate
200 tenacity
201 termcolor
202 typing-extensions
203 unicodecsv
204 werkzeug
205 ] ++ lib.optionals (pythonOlder "3.7") [
206 dataclasses
207 ] ++ lib.optionals (pythonOlder "3.9") [
208 importlib-metadata
209 ] ++ providerDependencies;
210
211 buildInputs = [
212 airflow-frontend
213 ];
214
215 checkInputs = [
216 freezegun
217 pytestCheckHook
218 ];
219
220 # By default, source code of providers is included but unusable due to missing
221 # transitive dependencies. To enable a provider, add it to extraProviders
222 # above
223 INSTALL_PROVIDERS_FROM_SOURCES = "true";
224
225 postPatch = ''
226 substituteInPlace setup.cfg \
227 --replace "colorlog>=4.0.2, <5.0" "colorlog" \
228 --replace "flask-login>=0.6.2" "flask-login" \
229 --replace "pathspec~=0.9.0" "pathspec"
230 '' + lib.optionalString stdenv.isDarwin ''
231 # Fix failing test on Hydra
232 substituteInPlace airflow/utils/db.py \
233 --replace "/tmp/sqlite_default.db" "$TMPDIR/sqlite_default.db"
234 '';
235
236 # allow for gunicorn processes to have access to Python packages
237 makeWrapperArgs = [
238 "--prefix PYTHONPATH : $PYTHONPATH"
239 ];
240
241 postInstall = ''
242 cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
243 # Needed for pythonImportsCheck below
244 export HOME=$(mktemp -d)
245 '';
246
247 pythonImportsCheck = [
248 "airflow"
249 ] ++ providerImports;
250
251 preCheck = ''
252 export AIRFLOW_HOME=$HOME
253 export AIRFLOW__CORE__UNIT_TEST_MODE=True
254 export AIRFLOW_DB="$HOME/airflow.db"
255 export PATH=$PATH:$out/bin
256
257 airflow version
258 airflow db init
259 airflow db reset -y
260 '';
261
262 pytestFlagsArray = [
263 "tests/core/test_core.py"
264 ];
265
266 disabledTests = lib.optionals stdenv.isDarwin [
267 "bash_operator_kill" # psutil.AccessDenied
268 ];
269
270 # Updates yarn.lock and package.json
271 passthru.updateScript = writeScript "update.sh" ''
272 #!/usr/bin/env nix-shell
273 #!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix
274
275 set -euo pipefail
276
277 # Get new version
278 new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html |
279 pcregrep -o1 'Airflow ([0-9.]+).' | head -1)"
280 update-source-version ${pname} "$new_version"
281
282 # Update frontend
283 cd ./pkgs/development/python-modules/apache-airflow
284 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
285 curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
286 # Note: for 2.3.4 a manual change was needed to get a fully resolved URL for
287 # caniuse-lite@1.0.30001312 (with the sha after the #). The error from yarn
288 # was 'Can't make a request in offline mode' from yarn. Corrected install by
289 # manually running yarn add caniuse-lite@1.0.30001312 and copying the
290 # requisite section from the generated yarn.lock.
291 yarn2nix > yarn.nix
292
293 # update provider dependencies
294 ./update-providers.py
295 '';
296
297 # Note on testing the web UI:
298 # You can (manually) test the web UI as follows:
299 #
300 # nix shell .#python3Packages.apache-airflow
301 # airflow db init
302 # airflow reset -y # WARNING: this will wipe any existing db state you might have!
303 # airflow standalone
304 #
305 # Then navigate to the localhost URL using the credentials printed, try
306 # triggering the 'example_bash_operator' and 'example_bash_operator' DAGs and
307 # see if they report success.
308
309 meta = with lib; {
310 description = "Programmatically author, schedule and monitor data pipelines";
311 homepage = "https://airflow.apache.org/";
312 license = licenses.asl20;
313 maintainers = with maintainers; [ bhipple gbpdt ingenieroariel ];
314 };
315}