tjh.dev
Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5 # build-system
6 setuptools,
7
8 # dependencies
9 dill,
10 filelock,
11 fsspec,
12 httpx,
13 huggingface-hub,
14 multiprocess,
15 numpy,
16 pandas,
17 pyarrow,
18 pyyaml,
19 requests,
20 tqdm,
21 xxhash,
22}:
23buildPythonPackage rec {
24 pname = "datasets";
25 version = "4.5.0";
26 pyproject = true;
27
28 src = fetchFromGitHub {
29 owner = "huggingface";
30 repo = "datasets";
31 tag = version;
32 hash = "sha256-K8JqIbYz3ZfT1t1h5dRGCo9kBQp0E+kElqzaw2InaOI=";
33 };
34
35 build-system = [
36 setuptools
37 ];
38
39 dependencies = [
40 dill
41 filelock
42 fsspec
43 httpx
44 huggingface-hub
45 multiprocess
46 numpy
47 pandas
48 pyarrow
49 pyyaml
50 requests
51 tqdm
52 xxhash
53 ]
54 ++ fsspec.optional-dependencies.http;
55
56 pythonRelaxDeps = [
57 # https://github.com/huggingface/datasets/blob/a256b85cbc67aa3f0e75d32d6586afc507cf535b/setup.py#L117
58 # "pin until dill has official support for determinism"
59 "dill"
60 # https://github.com/huggingface/datasets/blob/4.5.0/setup.py#L127
61 "multiprocess"
62 # https://github.com/huggingface/datasets/blob/4.5.0/setup.py#L130
63 "fsspec"
64 ];
65
66 # Tests require pervasive internet access
67 doCheck = false;
68
69 # Module import will attempt to create a cache directory
70 postFixup = "export HF_MODULES_CACHE=$TMPDIR";
71
72 pythonImportsCheck = [ "datasets" ];
73
74 meta = {
75 description = "Open-access datasets and evaluation metrics for natural language processing";
76 mainProgram = "datasets-cli";
77 homepage = "https://github.com/huggingface/datasets";
78 changelog = "https://github.com/huggingface/datasets/releases/tag/${src.tag}";
79 license = lib.licenses.asl20;
80 maintainers = with lib.maintainers; [ osbm ];
81 };
82}