tjh.dev
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 fetchFromGitHub,
6
7 # build-system
8 setuptools,
9
10 # dependencies
11 braceexpand,
12 numpy,
13 pyyaml,
14
15 # tests
16 imageio,
17 lmdb,
18 msgpack,
19 pytestCheckHook,
20 torch,
21 torchvision,
22}:
23buildPythonPackage rec {
24 pname = "webdataset";
25 version = "0.2.107";
26 pyproject = true;
27
28 src = fetchFromGitHub {
29 owner = "webdataset";
30 repo = "webdataset";
31 tag = "v${version}";
32 hash = "sha256-L9RUQItmW/7O/eTst2Sl/415EP4Jo662bKWbYA6p5bk=";
33 };
34
35 build-system = [
36 setuptools
37 ];
38
39 dependencies = [
40 braceexpand
41 numpy
42 pyyaml
43 ];
44
45 nativeCheckInputs = [
46 imageio
47 lmdb
48 msgpack
49 pytestCheckHook
50 torch
51 torchvision
52 ];
53
54 pythonImportsCheck = [ "webdataset" ];
55
56 preCheck = ''
57 export WIDS_CACHE=$TMPDIR
58 '';
59
60 disabledTests = [
61 # requires network
62 "test_batched"
63 "test_cache_dir"
64 "test_concurrent_download_and_open"
65 "test_dataloader"
66 "test_decode_handlers"
67 "test_decoder"
68 "test_download"
69 "test_handlers"
70 "test_pipe"
71 "test_remote_file"
72 "test_shard_syntax"
73 "test_torchvision"
74 "test_unbatched"
75 "test_yaml3"
76 ]
77 ++ lib.optionals stdenv.hostPlatform.isDarwin [
78 # pickling error
79 "test_background_download"
80 ]
81 ++ lib.optionals (stdenv.hostPlatform.isx86_64 && stdenv.hostPlatform.isDarwin) [
82 "test_concurrent_access"
83 ]
84 ++ lib.optionals (stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isLinux) [
85 # segfaults on aarch64-linux
86 "test_webloader"
87 "test_webloader2"
88 "test_webloader_repeat"
89 "test_webloader_unbatched"
90 ];
91
92 disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [
93 # Issue with creating a temp file in the sandbox
94 "tests/wids/test_wids_mmtar.py"
95 # hangs the build *after* the tests
96 "tests/webdataset/test_loaders.py"
97 ];
98
99 meta = {
100 description = "High-performance Python-based I/O system for large (and small) deep learning problems, with strong support for PyTorch";
101 mainProgram = "widsindex";
102 homepage = "https://github.com/webdataset/webdataset";
103 changelog = "https://github.com/webdataset/webdataset/releases/tag/${version}";
104 license = lib.licenses.bsd3;
105 maintainers = with lib.maintainers; [ iynaix ];
106 };
107}