tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 fetchFromGitHub,
6 rustPlatform,
7 pythonAtLeast,
8
9 # nativeBuildInputs
10 pkg-config,
11
12 # buildInputs
13 openssl,
14 protobuf,
15
16 # dependencies
17 lance-namespace,
18 numpy,
19 pyarrow,
20
21 # optional-dependencies
22 torch,
23
24 # tests
25 datafusion,
26 duckdb,
27 ml-dtypes,
28 pandas,
29 pillow,
30 polars,
31 pytestCheckHook,
32 tqdm,
33}:
34
35buildPythonPackage (finalAttrs: {
36 pname = "pylance";
37 version = "3.0.0";
38 pyproject = true;
39
40 src = fetchFromGitHub {
41 owner = "lancedb";
42 repo = "lance";
43 tag = "v${finalAttrs.version}";
44 hash = "sha256-71PogI877/dLwwlvMBraaC0vQWKtAHI/bmGEIBZVui4=";
45 };
46
47 sourceRoot = "${finalAttrs.src.name}/python";
48
49 cargoDeps = rustPlatform.fetchCargoVendor {
50 inherit (finalAttrs)
51 pname
52 version
53 src
54 sourceRoot
55 ;
56 hash = "sha256-scQDRyX3hweYZep+LVAsiVqOvDTEw/ss0/4M3R4ewDU=";
57 };
58
59 nativeBuildInputs = [
60 pkg-config
61 protobuf # for protoc
62 rustPlatform.cargoSetupHook
63 ];
64
65 build-system = [
66 rustPlatform.cargoSetupHook
67 rustPlatform.maturinBuildHook
68 ];
69
70 buildInputs = [
71 openssl
72 protobuf
73 ];
74
75 pythonRelaxDeps = [ "pyarrow" ];
76
77 dependencies = [
78 lance-namespace
79 numpy
80 pyarrow
81 ];
82
83 optional-dependencies = {
84 torch = [ torch ];
85 };
86
87 pythonImportsCheck = [ "lance" ];
88
89 nativeCheckInputs = [
90 datafusion
91 duckdb
92 ml-dtypes
93 pandas
94 pillow
95 polars
96 pytestCheckHook
97 tqdm
98 ]
99 ++ finalAttrs.passthru.optional-dependencies.torch;
100
101 preCheck = ''
102 cd python/tests
103 '';
104
105 pytestFlags = lib.optionals (pythonAtLeast "3.14") [
106 # DeprecationWarning: '_UnionGenericAlias' is deprecated and slated for removal in Python 3.17
107 "-Wignore::DeprecationWarning"
108 ];
109
110 disabledTestPaths = lib.optionals (pythonAtLeast "3.14") [
111 # RuntimeError: torch.compile is not supported on Python 3.14+
112 "torch_tests/test_bench_utils.py"
113 "torch_tests/test_distance.py"
114 "torch_tests/test_torch_kmeans.py"
115 ];
116
117 disabledTests = [
118 # Hangs indefinitely
119 "test_all_permutations"
120
121 # Writes to read-only build directory
122 "test_add_data_storage_version"
123 "test_fix_data_storage_version"
124 "test_fts_backward_v0_27_0"
125
126 # AttributeError: 'SessionContext' object has no attribute 'register_table_provider'
127 "test_table_loading"
128
129 # subprocess.CalledProcessError: Command ... returned non-zero exit status 1.
130 # ModuleNotFoundError: No module named 'lance'
131 "test_lance_log_file"
132 "test_lance_log_file_invalid_path"
133 "test_lance_log_file_with_directory_creation"
134 "test_timestamp_precision"
135 "test_tracing"
136
137 # Flaky (AssertionError)
138 "test_index_cache_size"
139
140 # OSError: LanceError(IO): Failed to initialize default tokenizer:
141 # An invalid argument was passed:
142 # 'LinderaError { kind: Parse, source: failed to build tokenizer: LinderaError(kind=Io, source=No such file or directory (os error 2)) }', /build/source/rust/lance-index/src/scalar/inverted/tokenizer/lindera.rs:63:21
143 "test_lindera_load_config_fallback"
144
145 # OSError: LanceError(IO): Failed to load tokenizer config
146 "test_indexed_filter_with_fts_index_with_lindera_ipadic_jp_tokenizer"
147 "test_lindera_ipadic_jp_tokenizer_bin_user_dict"
148 "test_lindera_ipadic_jp_tokenizer_csv_user_dict"
149 "test_lindera_load_config_priority"
150 ]
151 ++ lib.optionals (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) [
152 # OSError: LanceError(IO): Resources exhausted: Failed to allocate additional 1245184 bytes for ExternalSorter[0]...
153 "test_merge_insert_large"
154 ]
155 ++ lib.optionals stdenv.hostPlatform.isDarwin [
156 # Build hangs after all the tests are run due to a torch subprocess not exiting
157 "test_multiprocess_loading"
158
159 # torch._inductor.exc.InductorError: CppCompileError: C++ compile error
160 # OpenMP support not found
161 # TODO: figure out why this only happens on python 3.13 and not 3.14
162 "test_cosine_distance"
163 "test_ground_truth"
164 "test_index_cast_centroids"
165 "test_index_with_no_centroid_movement"
166 "test_l2_distance"
167 "test_l2_distance_f16_bf16_cpu"
168 "test_pairwise_cosine"
169 "test_torch_index_with_nans"
170 "test_torch_kmeans_nans"
171 ]
172 ++ lib.optionals (pythonAtLeast "3.14") [
173 # RuntimeError: torch.compile is not supported on Python 3.14+
174 "test_create_index_unsupported_accelerator"
175 "test_index_cast_centroids"
176 "test_index_with_no_centroid_movement"
177 "test_torch_index_with_nans"
178 ];
179
180 __darwinAllowLocalNetworking = true;
181
182 meta = {
183 description = "Python wrapper for Lance columnar format";
184 homepage = "https://github.com/lancedb/lance";
185 changelog = "https://github.com/lancedb/lance/releases/tag/${finalAttrs.src.tag}";
186 license = lib.licenses.asl20;
187 maintainers = with lib.maintainers; [ natsukium ];
188 };
189})