tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 fetchFromGitHub,
6
7 # build-system
8 setuptools,
9
10 # dependencies
11 braceexpand,
12 numpy,
13 pyyaml,
14
15 # tests
16 imageio,
17 lmdb,
18 msgpack,
19 pytestCheckHook,
20 torch,
21 torchvision,
22}:
23buildPythonPackage rec {
24 pname = "webdataset";
25 version = "0.2.107";
26 pyproject = true;
27
28 src = fetchFromGitHub {
29 owner = "webdataset";
30 repo = "webdataset";
31 tag = "v${version}";
32 hash = "sha256-L9RUQItmW/7O/eTst2Sl/415EP4Jo662bKWbYA6p5bk=";
33 };
34
35 build-system = [
36 setuptools
37 ];
38
39 dependencies = [
40 braceexpand
41 numpy
42 pyyaml
43 ];
44
45 nativeCheckInputs = [
46 imageio
47 lmdb
48 msgpack
49 pytestCheckHook
50 torch
51 torchvision
52 ];
53
54 pythonImportsCheck = [ "webdataset" ];
55
56 preCheck = ''
57 export WIDS_CACHE=$TMPDIR
58 '';
59
60 disabledTests =
61 [
62 # requires network
63 "test_batched"
64 "test_cache_dir"
65 "test_concurrent_download_and_open"
66 "test_dataloader"
67 "test_decode_handlers"
68 "test_decoder"
69 "test_download"
70 "test_handlers"
71 "test_pipe"
72 "test_remote_file"
73 "test_shard_syntax"
74 "test_torchvision"
75 "test_unbatched"
76 "test_yaml3"
77 ]
78 ++ lib.optionals stdenv.hostPlatform.isDarwin [
79 # pickling error
80 "test_background_download"
81 ]
82 ++ lib.optionals (stdenv.hostPlatform.isx86_64 && stdenv.hostPlatform.isDarwin) [
83 "test_concurrent_access"
84 ]
85 ++ lib.optionals (stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isLinux) [
86 # segfaults on aarch64-linux
87 "test_webloader"
88 "test_webloader2"
89 "test_webloader_repeat"
90 "test_webloader_unbatched"
91 ];
92
93 disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [
94 # Issue with creating a temp file in the sandbox
95 "tests/wids/test_wids_mmtar.py"
96 # hangs the build *after* the tests
97 "tests/webdataset/test_loaders.py"
98 ];
99
100 meta = {
101 description = "High-performance Python-based I/O system for large (and small) deep learning problems, with strong support for PyTorch";
102 mainProgram = "widsindex";
103 homepage = "https://github.com/webdataset/webdataset";
104 changelog = "https://github.com/webdataset/webdataset/releases/tag/${version}";
105 license = lib.licenses.bsd3;
106 maintainers = with lib.maintainers; [ iynaix ];
107 };
108}