tjh.dev
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 cargo,
6 cmake,
7 darwin,
8 fetchFromGitHub,
9 pkg-config,
10 pkgs, # zstd hidden by python3Packages.zstd
11 pytestCheckHook,
12 pytest-codspeed ? null, # Not in Nixpkgs
13 pytest-cov,
14 pytest-xdist,
15 pytest-benchmark,
16 rustc,
17 rustPlatform,
18 runCommand,
19
20 mimalloc,
21 jemalloc,
22 rust-jemalloc-sys,
23 # Another alternative is to try `mimalloc`
24 polarsMemoryAllocator ? mimalloc, # polarsJemalloc,
25 polarsJemalloc ?
26 let
27 jemalloc' = rust-jemalloc-sys.override {
28 jemalloc = jemalloc.override {
29 # "libjemalloc.so.2: cannot allocate memory in static TLS block"
30
31 # https://github.com/pola-rs/polars/issues/5401#issuecomment-1300998316
32 disableInitExecTls = true;
33 };
34 };
35 in
36 assert builtins.elem "--disable-initial-exec-tls" jemalloc'.configureFlags;
37 jemalloc',
38
39 polars,
40 python,
41}:
42
43let
44 version = "1.12.0";
45
46 # Hide symbols to prevent accidental use
47 rust-jemalloc-sys = throw "polars: use polarsMemoryAllocator over rust-jemalloc-sys";
48 jemalloc = throw "polars: use polarsMemoryAllocator over jemalloc";
49in
50
51buildPythonPackage {
52 pname = "polars";
53 inherit version;
54
55 src = fetchFromGitHub {
56 owner = "pola-rs";
57 repo = "polars";
58 rev = "py-${version}";
59 hash = "sha256-q//vt8FvVKY9N/BOIoOwxaSB/F/tNX1Zl/9jd0AzSH4=";
60 };
61
62 # Do not type-check assertions because some of them use unstable features (`is_none_or`)
63 postPatch = ''
64 while IFS= read -r -d "" path ; do
65 sed -i 's \(\s*\)debug_assert! \1#[cfg(debug_assertions)]\n\1debug_assert! ' "$path"
66 done < <( find -iname '*.rs' -print0 )
67 '';
68
69 cargoDeps = rustPlatform.importCargoLock {
70 lockFile = ./Cargo.lock;
71 outputHashes = {
72 "numpy-0.21.0" = "sha256-u0Z+6L8pXSPaA3cE1sUpY6sCoaU1clXUcj/avnNzmsw=";
73 "polars-parquet-format-2.10.0" = "sha256-iB3KZ72JSp7tJCLn9moukpDEGf9MUos04rIQ9rDGWfI=";
74 };
75 };
76
77 requiredSystemFeatures = [ "big-parallel" ];
78
79 build-system = [ rustPlatform.maturinBuildHook ];
80
81 nativeBuildInputs = [
82 cargo
83 pkg-config
84 cmake # libz-ng-sys
85 rustPlatform.cargoSetupHook
86 rustPlatform.cargoBuildHook
87 rustPlatform.cargoInstallHook
88 rustc
89 ];
90
91 buildInputs =
92 [
93 polarsMemoryAllocator
94 (pkgs.__splicedPackages.zstd or pkgs.zstd)
95 ]
96 ++ lib.optionals stdenv.hostPlatform.isDarwin [
97 darwin.apple_sdk.frameworks.AppKit
98 darwin.apple_sdk.frameworks.IOKit
99 darwin.apple_sdk.frameworks.Security
100 ];
101
102 env = {
103 ZSTD_SYS_USE_PKG_CONFIG = true;
104
105 # https://github.com/NixOS/nixpkgs/blob/5c38beb516f8da3a823d94b746dd3bf3c6b9bbd7/doc/languages-frameworks/rust.section.md#using-community-maintained-rust-toolchains-using-community-maintained-rust-toolchains
106 # https://discourse.nixos.org/t/nixpkgs-rustplatform-and-nightly/22870
107 RUSTC_BOOTSTRAP = true;
108
109 # Several `debug_assert!` statements use the unstable `Option::is_none_or` method
110 RUSTFLAGS = lib.concatStringsSep " " (
111 [
112 "-Cdebug_assertions=n"
113 ]
114 ++ lib.optionals (polarsMemoryAllocator.pname == "mimalloc") [
115 "--cfg use_mimalloc"
116 ]
117 );
118 RUST_BACKTRACE = true;
119 };
120
121 dontUseCmakeConfigure = true;
122
123 maturinBuildFlags = [
124 "-m"
125 "py-polars/Cargo.toml"
126 ];
127
128 postInstall = ''
129 # Move polars.abi3.so -> polars.so
130 local polarsSo=""
131 local soName=""
132 while IFS= read -r -d "" p ; do
133 polarsSo=$p
134 soName="$(basename "$polarsSo")"
135 [[ "$soName" == polars.so ]] && break
136 done < <( find "$out" -iname "polars*.so" -print0 )
137 [[ -z "''${polarsSo:-}" ]] && echo "polars.so not found" >&2 && exit 1
138 if [[ "$soName" != polars.so ]] ; then
139 mv "$polarsSo" "$(dirname "$polarsSo")/polars.so"
140 fi
141 '';
142
143 pythonImportsCheck = [
144 "polars"
145 ];
146
147 passthru.tests.dynloading-1 =
148 runCommand "polars-dynloading-1"
149 {
150 nativeBuildInputs = [
151 (python.withPackages (ps: [
152 ps.pyarrow
153 polars
154 ]))
155 ];
156 }
157 ''
158 ((LD_DEBUG=libs python) |& tee $out | tail) << \EOF
159 import pyarrow
160 import polars
161 EOF
162 touch $out
163 '';
164 passthru.tests.dynloading-2 =
165 runCommand "polars-dynloading-2"
166 {
167 nativeBuildInputs = [
168 (python.withPackages (ps: [
169 ps.pyarrow
170 polars
171 ]))
172 ];
173 failureHook = ''
174 sed "s/^/ /" $out >&2
175 '';
176 }
177 ''
178 ((LD_DEBUG=libs python) |& tee $out | tail) << \EOF
179 import polars
180 import pyarrow
181 EOF
182 '';
183 passthru.tests.pytest = stdenv.mkDerivation {
184 pname = "${polars.pname}-pytest";
185
186 inherit (polars) version src;
187
188 requiredSystemFeatures = [ "big-parallel" ];
189
190 sourceRoot = "source/py-polars";
191 postPatch = ''
192 for f in * ; do
193 [[ "$f" == "tests" ]] || \
194 [[ "$f" == "pyproject.toml" ]] || \
195 rm -rf "$f"
196 done
197 for pat in "__pycache__" "*.pyc" ; do
198 find -iname "$pat" -exec rm "{}" ";"
199 done
200 '';
201 dontConfigure = true;
202 dontBuild = true;
203
204 doCheck = true;
205 checkPhase = "pytestCheckPhase";
206 nativeBuildInputs = [
207 (python.withPackages (ps: [
208 polars
209 ps.aiosqlite
210 ps.altair
211 ps.boto3
212 ps.deltalake
213 ps.flask
214 ps.flask-cors
215 ps.fsspec
216 ps.gevent
217 ps.hypothesis
218 ps.jax
219 ps.jaxlib
220 (ps.kuzu or null)
221 ps.moto
222 ps.nest-asyncio
223 ps.numpy
224 ps.openpyxl
225 ps.pandas
226 ps.pyarrow
227 ps.pydantic
228 (ps.pyiceberg or null)
229 ps.sqlalchemy
230 ps.torch
231 ps.xlsx2csv
232 ps.xlsxwriter
233 ps.zstandard
234 ps.cloudpickle
235 ]))
236 ];
237 nativeCheckInputs = [
238 pytestCheckHook
239 pytest-codspeed
240 pytest-cov
241 pytest-xdist
242 pytest-benchmark
243 ];
244
245 pytestFlagsArray = [
246 "-n auto"
247 "--dist loadgroup"
248 ''-m "slow or not slow"''
249 ];
250 disabledTests = [
251 "test_read_kuzu_graph_database" # kuzu
252 "test_read_database_cx_credentials" # connectorx
253
254 # adbc_driver_.*
255 "test_write_database_append_replace"
256 "test_write_database_create"
257 "test_write_database_create_quoted_tablename"
258 "test_write_database_adbc_temporary_table"
259 "test_write_database_create"
260 "test_write_database_append_replace"
261 "test_write_database_errors"
262 "test_write_database_errors"
263 "test_write_database_create_quoted_tablename"
264
265 # Internet access:
266 "test_read_web_file"
267
268 # Untriaged
269 "test_pickle_lazyframe_nested_function_udf"
270 "test_serde_udf"
271 "test_hash_struct"
272 ];
273 disabledTestPaths = [
274 "tests/benchmark"
275 "tests/docs"
276
277 "tests/unit/io/test_iceberg.py" # Package pyiceberg
278 "tests/unit/io/test_spreadsheet.py" # Package fastexcel
279
280 # Wrong altair version
281 "tests/unit/operations/namespaces/test_plot.py"
282
283 # adbc
284 "tests/unit/io/database/test_read.py"
285
286 # Untriaged
287 "tests/unit/cloud/test_prepare_cloud_plan.py"
288 "tests/unit/io/cloud/test_cloud.py"
289 ];
290
291 installPhase = "touch $out";
292 };
293
294 meta = {
295 description = "Dataframes powered by a multithreaded, vectorized query engine, written in Rust";
296 homepage = "https://github.com/pola-rs/polars";
297 license = lib.licenses.mit;
298 maintainers = with lib.maintainers; [
299 happysalada
300 SomeoneSerge
301 ];
302 mainProgram = "polars";
303 platforms = lib.platforms.all;
304 };
305}