tjh.dev
1{
2 lib,
3 stdenv,
4 appdirs,
5 azure-containerregistry,
6 azure-core,
7 azure-identity,
8 azure-storage-blob,
9 bokeh,
10 boto3,
11 buildPythonPackage,
12 click,
13 docker-pycreds,
14 fetchFromGitHub,
15 flask,
16 git,
17 gitpython,
18 google-cloud-artifact-registry,
19 google-cloud-compute,
20 google-cloud-storage,
21 hypothesis,
22 imageio,
23 jsonref,
24 jsonschema,
25 keras,
26 kubernetes,
27 matplotlib,
28 mlflow,
29 moviepy,
30 nbclient,
31 nbformat,
32 pandas,
33 parameterized,
34 pathtools,
35 protobuf,
36 psutil,
37 pydantic,
38 pyfakefs,
39 pytest-mock,
40 pytest-xdist,
41 pytestCheckHook,
42 pythonOlder,
43 pyyaml,
44 requests,
45 responses,
46 scikit-learn,
47 sentry-sdk,
48 setproctitle,
49 setuptools,
50 soundfile,
51 substituteAll,
52 torch,
53 tqdm,
54}:
55
56buildPythonPackage rec {
57 pname = "wandb";
58 version = "0.16.0";
59 format = "pyproject";
60
61 disabled = pythonOlder "3.6";
62
63 src = fetchFromGitHub {
64 owner = pname;
65 repo = pname;
66 rev = "refs/tags/v${version}";
67 hash = "sha256-XXs9KjiAPzZ932r4UJ87RpM+qhg/bNDWEYsq2Ua6SRw=";
68 };
69
70 patches = [
71 # Replace git paths
72 (substituteAll {
73 src = ./hardcode-git-path.patch;
74 git = "${lib.getBin git}/bin/git";
75 })
76 ];
77
78 nativeBuildInputs = [
79 setuptools
80 ];
81
82 # setuptools is necessary since pkg_resources is required at runtime.
83 propagatedBuildInputs = [
84 appdirs
85 click
86 docker-pycreds
87 gitpython
88 pathtools
89 protobuf
90 psutil
91 pyyaml
92 requests
93 sentry-sdk
94 setproctitle
95 setuptools
96 ];
97
98 __darwinAllowLocalNetworking = true;
99
100 nativeCheckInputs = [
101 azure-containerregistry
102 azure-core
103 azure-identity
104 azure-storage-blob
105 bokeh
106 boto3
107 flask
108 google-cloud-artifact-registry
109 google-cloud-compute
110 google-cloud-storage
111 hypothesis
112 imageio
113 jsonref
114 jsonschema
115 keras
116 kubernetes
117 matplotlib
118 mlflow
119 moviepy
120 nbclient
121 nbformat
122 pandas
123 parameterized
124 pydantic
125 pyfakefs
126 pytest-mock
127 pytest-xdist
128 pytestCheckHook
129 responses
130 scikit-learn
131 soundfile
132 torch
133 tqdm
134 ];
135
136 # Set BOKEH_CDN_VERSION to stop bokeh throwing an exception in tests
137 preCheck = ''
138 export HOME=$(mktemp -d)
139 export BOKEH_CDN_VERSION=${bokeh.version}
140 '';
141
142 pythonRelaxDeps = [ "protobuf" ];
143
144 pytestFlagsArray = [
145 # We want to run only unit tests
146 "tests/pytest_tests"
147 ];
148
149 disabledTestPaths =
150 [
151 # Tests that try to get chatty over sockets or spin up servers, not possible in the nix build environment.
152 "tests/pytest_tests/system_tests/test_notebooks/test_notebooks.py"
153 "tests/pytest_tests/unit_tests_old/test_cli.py"
154 "tests/pytest_tests/unit_tests_old/test_data_types.py"
155 "tests/pytest_tests/unit_tests_old/test_file_stream.py"
156 "tests/pytest_tests/unit_tests_old/test_file_upload.py"
157 "tests/pytest_tests/unit_tests_old/test_footer.py"
158 "tests/pytest_tests/unit_tests_old/test_internal_api.py"
159 "tests/pytest_tests/unit_tests_old/test_metric_internal.py"
160 "tests/pytest_tests/unit_tests_old/test_public_api.py"
161 "tests/pytest_tests/unit_tests_old/test_runtime.py"
162 "tests/pytest_tests/unit_tests_old/test_sender.py"
163 "tests/pytest_tests/unit_tests_old/test_summary.py"
164 "tests/pytest_tests/unit_tests_old/test_tb_watcher.py"
165 "tests/pytest_tests/unit_tests_old/test_time_resolution.py"
166 "tests/pytest_tests/unit_tests_old/test_wandb_agent.py"
167 "tests/pytest_tests/unit_tests_old/test_wandb_integration.py"
168 "tests/pytest_tests/unit_tests_old/test_wandb_run.py"
169 "tests/pytest_tests/unit_tests_old/test_wandb.py"
170 "tests/pytest_tests/unit_tests_old/tests_launch/test_launch_aws.py"
171 "tests/pytest_tests/unit_tests_old/tests_launch/test_launch_cli.py"
172 "tests/pytest_tests/unit_tests_old/tests_launch/test_launch_docker.py"
173 "tests/pytest_tests/unit_tests_old/tests_launch/test_launch.py"
174 "tests/pytest_tests/unit_tests/test_cli.py"
175 "tests/pytest_tests/unit_tests/test_data_types.py"
176 "tests/pytest_tests/unit_tests/test_internal_api.py"
177 "tests/pytest_tests/unit_tests/test_mode_disabled.py"
178 "tests/pytest_tests/unit_tests/test_model_workflows.py"
179 "tests/pytest_tests/unit_tests/test_plots.py"
180 "tests/pytest_tests/unit_tests/test_public_api.py"
181 "tests/pytest_tests/unit_tests/test_sender.py"
182 "tests/pytest_tests/unit_tests/test_util.py"
183 "tests/pytest_tests/unit_tests/test_wandb_verify.py"
184
185 # Requires docker access
186 "tests/pytest_tests/system_tests/test_artifacts/test_artifact_saver.py"
187 "tests/pytest_tests/system_tests/test_artifacts/test_misc.py"
188 "tests/pytest_tests/system_tests/test_artifacts/test_misc2.py"
189 "tests/pytest_tests/system_tests/test_artifacts/test_object_references.py"
190 "tests/pytest_tests/system_tests/test_artifacts/test_wandb_artifacts_full.py"
191 "tests/pytest_tests/system_tests/test_artifacts/test_wandb_artifacts.py"
192 "tests/pytest_tests/system_tests/test_core/test_cli_full.py"
193 "tests/pytest_tests/system_tests/test_core/test_data_types_full.py"
194 "tests/pytest_tests/system_tests/test_core/test_file_stream_internal.py"
195 "tests/pytest_tests/system_tests/test_core/test_file_upload.py"
196 "tests/pytest_tests/system_tests/test_core/test_footer.py"
197 "tests/pytest_tests/system_tests/test_core/test_keras_full.py"
198 "tests/pytest_tests/system_tests/test_core/test_label_full.py"
199 "tests/pytest_tests/system_tests/test_core/test_metric_full.py"
200 "tests/pytest_tests/system_tests/test_core/test_metric_internal.py"
201 "tests/pytest_tests/system_tests/test_core/test_mode_disabled_full.py"
202 "tests/pytest_tests/system_tests/test_core/test_model_workflow.py"
203 "tests/pytest_tests/system_tests/test_core/test_mp_full.py"
204 "tests/pytest_tests/system_tests/test_core/test_public_api.py"
205 "tests/pytest_tests/system_tests/test_core/test_redir_full.py"
206 "tests/pytest_tests/system_tests/test_core/test_report_api.py"
207 "tests/pytest_tests/system_tests/test_core/test_save_policies.py"
208 "tests/pytest_tests/system_tests/test_core/test_sender.py"
209 "tests/pytest_tests/system_tests/test_core/test_start_method.py"
210 "tests/pytest_tests/system_tests/test_core/test_system_info.py"
211 "tests/pytest_tests/system_tests/test_core/test_tb_watcher.py"
212 "tests/pytest_tests/system_tests/test_core/test_telemetry_full.py"
213 "tests/pytest_tests/system_tests/test_core/test_time_resolution.py"
214 "tests/pytest_tests/system_tests/test_core/test_torch_full.py"
215 "tests/pytest_tests/system_tests/test_core/test_validation_data_logger.py"
216 "tests/pytest_tests/system_tests/test_core/test_wandb_init.py"
217 "tests/pytest_tests/system_tests/test_core/test_wandb_integration.py"
218 "tests/pytest_tests/system_tests/test_core/test_wandb_run.py"
219 "tests/pytest_tests/system_tests/test_core/test_wandb_settings.py"
220 "tests/pytest_tests/system_tests/test_core/test_wandb_tensorflow.py"
221 "tests/pytest_tests/system_tests/test_core/test_wandb_verify.py"
222 "tests/pytest_tests/system_tests/test_core/test_wandb.py"
223 "tests/pytest_tests/system_tests/test_importers/test_import_mlflow.py"
224 "tests/pytest_tests/system_tests/test_launch/test_github_reference.py"
225 "tests/pytest_tests/system_tests/test_launch/test_job_status_tracker.py"
226 "tests/pytest_tests/system_tests/test_launch/test_job.py"
227 "tests/pytest_tests/system_tests/test_launch/test_launch_add.py"
228 "tests/pytest_tests/system_tests/test_launch/test_launch_cli.py"
229 "tests/pytest_tests/system_tests/test_launch/test_launch_kubernetes.py"
230 "tests/pytest_tests/system_tests/test_launch/test_launch_local_container.py"
231 "tests/pytest_tests/system_tests/test_launch/test_launch_run.py"
232 "tests/pytest_tests/system_tests/test_launch/test_launch_sagemaker.py"
233 "tests/pytest_tests/system_tests/test_launch/test_launch_sweep_cli.py"
234 "tests/pytest_tests/system_tests/test_launch/test_launch_sweep.py"
235 "tests/pytest_tests/system_tests/test_launch/test_launch_vertex.py"
236 "tests/pytest_tests/system_tests/test_launch/test_launch.py"
237 "tests/pytest_tests/system_tests/test_launch/test_wandb_reference.py"
238 "tests/pytest_tests/system_tests/test_nexus/test_nexus.py"
239 "tests/pytest_tests/system_tests/test_sweep/test_public_api.py"
240 "tests/pytest_tests/system_tests/test_sweep/test_sweep_scheduler.py"
241 "tests/pytest_tests/system_tests/test_sweep/test_sweep_utils.py"
242 "tests/pytest_tests/system_tests/test_sweep/test_wandb_agent_full.py"
243 "tests/pytest_tests/system_tests/test_sweep/test_wandb_agent.py"
244 "tests/pytest_tests/system_tests/test_sweep/test_wandb_sweep.py"
245 "tests/pytest_tests/system_tests/test_system_metrics/test_open_metrics.py"
246 "tests/pytest_tests/system_tests/test_system_metrics/test_system_monitor.py"
247
248 # Tries to access /homeless-shelter
249 "tests/pytest_tests/unit_tests/test_tables.py"
250
251 # E AssertionError: assert 'Cannot use both --async and --queue with wandb launch' in 'wandb: ERROR Find detailed error logs at: /build/source/wandb/debug-cli.nixbld.log\nError: The wandb service process exited with 1. Ensure that `sys.executable` is a valid python interpreter. You can override it with the `_executable` setting or with the `WANDB__EXECUTABLE` environment variable.\n'
252 # E + where 'wandb: ERROR Find detailed error logs at: /build/source/wandb/debug-cli.nixbld.log\nError: The wandb service process exited with 1. Ensure that `sys.executable` is a valid python interpreter. You can override it with the `_executable` setting or with the `WANDB__EXECUTABLE` environment variable.\n' = <Result SystemExit(1)>.output
253 "tests/pytest_tests/unit_tests_old/tests_launch/test_launch_jobs.py"
254
255 # Requires google-cloud-aiplatform which is not packaged as of 2023-04-25.
256 "tests/pytest_tests/unit_tests/test_launch/test_runner/test_vertex.py"
257
258 # Requires google-cloud-artifact-registry which is not packaged as of 2023-04-25.
259 "tests/pytest_tests/unit_tests/test_launch/test_registry/test_gcp_artifact_registry.py"
260
261 # Requires kfp which is not packaged as of 2023-04-25.
262 "tests/pytest_tests/system_tests/test_core/test_kfp.py"
263
264 # Requires kubernetes_asyncio which is not packaged as of 2024-01-14.
265 "tests/pytest_tests/unit_tests/test_launch/test_builder/test_kaniko.py"
266 "tests/pytest_tests/unit_tests/test_launch/test_runner/test_kubernetes.py"
267 "tests/pytest_tests/unit_tests/test_launch/test_runner/test_safe_watch.py"
268
269 # Requires metaflow which is not packaged as of 2023-04-25.
270 "tests/pytest_tests/unit_tests/test_metaflow.py"
271
272 # Requires tensorflow which is broken as of 2023-09-03
273 "tests/pytest_tests/unit_tests/test_keras.py"
274
275 # Try to get hardware information, not possible in the nix build environment
276 "tests/pytest_tests/unit_tests/test_system_metrics/test_disk.py"
277
278 # See https://github.com/wandb/wandb/issues/5423
279 "tests/pytest_tests/unit_tests/test_docker.py"
280 "tests/pytest_tests/unit_tests/test_library_public.py"
281
282 # See https://github.com/wandb/wandb/issues/6836
283 "tests/pytest_tests/unit_tests_old/test_logging.py"
284 ]
285 ++ lib.optionals stdenv.isLinux [
286 # Same as above
287 "tests/pytest_tests/unit_tests/test_artifacts/test_storage.py"
288 ]
289 ++ lib.optionals stdenv.isDarwin [
290 # Same as above
291 "tests/pytest_tests/unit_tests/test_lib/test_filesystem.py"
292 ];
293
294 disabledTests =
295 [
296 # Timing sensitive
297 "test_login_timeout"
298
299 # Tensorflow 2.13 is too old for the current version of keras
300 # ModuleNotFoundError: No module named 'keras.api._v2'
301 "test_saved_model_keras"
302 "test_sklearn_saved_model"
303 "test_pytorch_saved_model"
304 "test_tensorflow_keras_saved_model"
305 ]
306 ++ lib.optionals stdenv.isDarwin [
307 # Disable test that fails on darwin due to issue with python3Packages.psutil:
308 # https://github.com/giampaolo/psutil/issues/1219
309 "test_tpu_system_stats"
310 ];
311
312 pythonImportsCheck = [ "wandb" ];
313
314 # unmaintainable list of disabled tests
315 passthru.skipBulkUpdate = true;
316
317 meta = with lib; {
318 description = "CLI and library for interacting with the Weights and Biases API";
319 homepage = "https://github.com/wandb/wandb";
320 changelog = "https://github.com/wandb/wandb/raw/v${version}/CHANGELOG.md";
321 license = licenses.mit;
322 maintainers = with maintainers; [ samuela ];
323 };
324}