tjh.dev
1{ lib
2, buildPythonPackage
3, fetchFromGitHub
4, fetchpatch
5, pythonOlder
6, pytestCheckHook
7, atpublic
8, bidict
9, black
10, clickhouse-cityhash
11, clickhouse-driver
12, dask
13, datafusion
14, db-dtypes
15, duckdb
16, duckdb-engine
17, filelock
18, geoalchemy2
19, geopandas
20, google-cloud-bigquery
21, google-cloud-bigquery-storage
22, graphviz-nox
23, hypothesis
24, importlib-resources
25, lz4
26, multipledispatch
27, numpy
28, packaging
29, pandas
30, parsy
31, poetry-core
32, polars
33, pooch
34, psycopg2
35, pyarrow
36, pydata-google-auth
37, pydruid
38, pymysql
39, pyspark
40, pytest-benchmark
41, pytest-httpserver
42, pytest-mock
43, pytest-randomly
44, pytest-snapshot
45, pytest-xdist
46, python-dateutil
47, pytz
48, regex
49, rich
50, rsync
51, shapely
52, snowflake-connector-python
53, snowflake-sqlalchemy
54, sqlalchemy
55, sqlalchemy-views
56, sqlglot
57, sqlite
58, toolz
59, trino-python-client
60, typing-extensions
61}:
62let
63 testBackends = [ "datafusion" "duckdb" "pandas" "sqlite" ];
64
65 ibisTestingData = fetchFromGitHub {
66 name = "ibis-testing-data";
67 owner = "ibis-project";
68 repo = "testing-data";
69 rev = "8a59df99c01fa217259554929543e71c3bbb1761";
70 hash = "sha256-NbgEe0w/qf9hCr9rRfIpyaH9pv25I8x0ykY7EJxDOuk=";
71 };
72in
73
74buildPythonPackage rec {
75 pname = "ibis-framework";
76 version = "5.1.0";
77 format = "pyproject";
78
79 disabled = pythonOlder "3.8";
80
81 src = fetchFromGitHub {
82 name = "ibis-source";
83 repo = "ibis";
84 owner = "ibis-project";
85 rev = "refs/tags/${version}";
86 hash = "sha256-u3BBGdhWajZ5WtoBvNxmx76+orfHY6LX3IWAq/x2/9A=";
87 };
88
89 patches = [
90 # fixes a small bug in the datafusion backend to reorder predicates
91 (fetchpatch {
92 name = "fix-datafusion-compilation.patch";
93 url = "https://github.com/ibis-project/ibis/commit/009230421b2bc1f86591e8b850d37a489e8e4f06.patch";
94 hash = "sha256-5NHkgc8d2bkOMpbY1vme1XgNfyHSr0f7BrR3JTTjjPI=";
95 })
96 ];
97
98 nativeBuildInputs = [
99 poetry-core
100 ];
101
102 propagatedBuildInputs = [
103 atpublic
104 bidict
105 multipledispatch
106 numpy
107 pandas
108 parsy
109 pooch
110 python-dateutil
111 pytz
112 rich
113 sqlglot
114 toolz
115 typing-extensions
116 ] ++ lib.optionals (pythonOlder "3.9") [ importlib-resources ]
117 ++ pooch.optional-dependencies.progress
118 ++ pooch.optional-dependencies.xxhash;
119
120 nativeCheckInputs = [
121 pytestCheckHook
122 filelock
123 hypothesis
124 pytest-benchmark
125 pytest-httpserver
126 pytest-mock
127 pytest-randomly
128 pytest-snapshot
129 pytest-xdist
130 rsync
131 ] ++ lib.concatMap (name: passthru.optional-dependencies.${name}) testBackends;
132
133 pytestFlagsArray = [
134 "--dist=loadgroup"
135 "-m"
136 "'${lib.concatStringsSep " or " testBackends} or core'"
137 # sqlalchemy2 breakage
138 "--deselect=ibis/tests/sql/test_sqlalchemy.py::test_tpc_h17"
139 # tries to download duckdb extensions
140 "--deselect=ibis/backends/duckdb/tests/test_register.py::test_register_sqlite"
141 "--deselect=ibis/backends/duckdb/tests/test_register.py::test_read_sqlite"
142 ];
143
144 # patch out tests that check formatting with black
145 postPatch = ''
146 find ibis/tests -type f -name '*.py' -exec sed -i \
147 -e '/^ *assert_decompile_roundtrip/d' \
148 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
149 '';
150
151 preCheck = ''
152 set -eo pipefail
153
154 HOME="$TMPDIR"
155 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
156
157 mkdir -p "$IBIS_TEST_DATA_DIRECTORY"
158
159 # copy the test data to a directory
160 rsync --chmod=Du+rwx,Fu+rw --archive "${ibisTestingData}/" "$IBIS_TEST_DATA_DIRECTORY"
161 '';
162
163 postCheck = ''
164 rm -r "$IBIS_TEST_DATA_DIRECTORY"
165 '';
166
167 pythonImportsCheck = [
168 "ibis"
169 ] ++ map (backend: "ibis.backends.${backend}") testBackends;
170
171 passthru = {
172 optional-dependencies = {
173 bigquery = [ db-dtypes google-cloud-bigquery google-cloud-bigquery-storage pydata-google-auth ];
174 clickhouse = [ clickhouse-cityhash clickhouse-driver lz4 sqlalchemy ];
175 dask = [ dask pyarrow regex ];
176 datafusion = [ datafusion ];
177 druid = [ pydruid sqlalchemy ];
178 duckdb = [ duckdb duckdb-engine packaging pyarrow sqlalchemy sqlalchemy-views ];
179 geospatial = [ geoalchemy2 geopandas shapely ];
180 mysql = [ sqlalchemy pymysql sqlalchemy-views ];
181 pandas = [ regex ];
182 polars = [ polars pyarrow ];
183 postgres = [ psycopg2 sqlalchemy sqlalchemy-views ];
184 pyspark = [ pyarrow pyspark sqlalchemy ];
185 snowflake = [ snowflake-connector-python snowflake-sqlalchemy sqlalchemy-views ];
186 sqlite = [ regex sqlalchemy sqlite sqlalchemy-views ];
187 trino = [ trino-python-client sqlalchemy sqlalchemy-views ];
188 visualization = [ graphviz-nox ];
189 decompiler = [ black ];
190 };
191 };
192
193 meta = with lib; {
194 description = "Productivity-centric Python Big Data Framework";
195 homepage = "https://github.com/ibis-project/ibis";
196 changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
197 license = licenses.asl20;
198 maintainers = with maintainers; [ costrouc cpcloud ];
199 };
200}