tjh.dev
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 fetchFromGitHub,
6 rustPlatform,
7 pytestCheckHook,
8 libiconv,
9 numpy,
10 protobuf,
11 protoc,
12 pyarrow,
13 Security,
14 SystemConfiguration,
15 typing-extensions,
16 pythonOlder,
17}:
18
19let
20 arrow-testing = fetchFromGitHub {
21 name = "arrow-testing";
22 owner = "apache";
23 repo = "arrow-testing";
24 rev = "4d209492d514c2d3cb2d392681b9aa00e6d8da1c";
25 hash = "sha256-IkiCbuy0bWyClPZ4ZEdkEP7jFYLhM7RCuNLd6Lazd4o=";
26 };
27
28 parquet-testing = fetchFromGitHub {
29 name = "parquet-testing";
30 owner = "apache";
31 repo = "parquet-testing";
32 rev = "50af3d8ce206990d81014b1862e5ce7380dc3e08";
33 hash = "sha256-edyv/r5olkj09aHtm8LHZY0b3jUtLNUcufwI41qKYaY=";
34 };
35in
36
37buildPythonPackage rec {
38 pname = "datafusion";
39 version = "40.1.0";
40 pyproject = true;
41
42 src = fetchFromGitHub {
43 name = "datafusion-source";
44 owner = "apache";
45 repo = "arrow-datafusion-python";
46 rev = "refs/tags/${version}";
47 hash = "sha256-5WOSlx4XW9zO6oTY16lWQElShLv0ubflVPfSSEGrFgg=";
48 };
49
50 cargoDeps = rustPlatform.fetchCargoTarball {
51 name = "datafusion-cargo-deps";
52 inherit src;
53 hash = "sha256-hN03tbnH77VsMDxSMddMHIH00t7lUs5h8rTHbiMIExw=";
54 };
55
56 nativeBuildInputs = with rustPlatform; [
57 cargoSetupHook
58 maturinBuildHook
59 protoc
60 ];
61
62 buildInputs =
63 [ protobuf ]
64 ++ lib.optionals stdenv.hostPlatform.isDarwin [
65 libiconv
66 Security
67 SystemConfiguration
68 ];
69
70 dependencies = [
71 pyarrow
72 ] ++ lib.optionals (pythonOlder "3.13") [ typing-extensions ];
73
74 nativeCheckInputs = [
75 pytestCheckHook
76 numpy
77 ];
78
79 pythonImportsCheck = [ "datafusion" ];
80
81 pytestFlagsArray = [
82 "--pyargs"
83 pname
84 ];
85
86 preCheck = ''
87 pushd $TMPDIR
88 ln -s ${arrow-testing} ./testing
89 ln -s ${parquet-testing} ./parquet
90 '';
91
92 postCheck = ''
93 popd
94 '';
95
96 meta = with lib; {
97 description = "Extensible query execution framework";
98 longDescription = ''
99 DataFusion is an extensible query execution framework, written in Rust,
100 that uses Apache Arrow as its in-memory format.
101 '';
102 homepage = "https://arrow.apache.org/datafusion/";
103 changelog = "https://github.com/apache/arrow-datafusion-python/blob/${version}/CHANGELOG.md";
104 license = with licenses; [ asl20 ];
105 maintainers = with maintainers; [ cpcloud ];
106 };
107}