tjh.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5 pythonOlder,
6 pytestCheckHook,
7 atpublic,
8 black,
9 clickhouse-connect,
10 datafusion,
11 db-dtypes,
12 duckdb,
13 fetchpatch,
14 filelock,
15 geopandas,
16 google-cloud-bigquery,
17 google-cloud-bigquery-storage,
18 graphviz,
19 hypothesis,
20 numpy,
21 oracledb,
22 packaging,
23 pandas,
24 parsy,
25 pins,
26 poetry-core,
27 poetry-dynamic-versioning,
28 polars,
29 psycopg2,
30 pyarrow,
31 pyarrow-hotfix,
32 pydata-google-auth,
33 pydruid,
34 pymysql,
35 pyodbc,
36 pyspark,
37 pytest-benchmark,
38 pytest-httpserver,
39 pytest-mock,
40 pytest-randomly,
41 pytest-snapshot,
42 pytest-timeout,
43 pytest-xdist,
44 python-dateutil,
45 pytz,
46 regex,
47 rich,
48 shapely,
49 snowflake-connector-python,
50 sqlglot,
51 sqlite,
52 toolz,
53 trino-python-client,
54 typing-extensions,
55}:
56let
57 testBackends = [
58 "duckdb"
59 "sqlite"
60 ];
61
62 ibisTestingData = fetchFromGitHub {
63 name = "ibis-testing-data";
64 owner = "ibis-project";
65 repo = "testing-data";
66 # https://github.com/ibis-project/ibis/blob/9.5.0/nix/overlay.nix#L20-L26
67 rev = "b26bd40cf29004372319df620c4bbe41420bb6f8";
68 hash = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU=";
69 };
70in
71
72buildPythonPackage rec {
73 pname = "ibis-framework";
74 version = "9.5.0";
75 pyproject = true;
76
77 disabled = pythonOlder "3.10";
78
79 src = fetchFromGitHub {
80 name = "ibis-source";
81 repo = "ibis";
82 owner = "ibis-project";
83 rev = "refs/tags/${version}";
84 hash = "sha256-6ebw/E3jZFMHKqC5ZY//2Ke0NrklyoGp5JGKBfDxy40=";
85 };
86
87 patches = [
88 # remove after the 10.0 release
89 (fetchpatch {
90 name = "ibis-framework-duckdb-1.1.1.patch";
91 url = "https://github.com/ibis-project/ibis/commit/a54eceabac1d6592e9f6ab0ca7749e37a748c2ad.patch";
92 hash = "sha256-j5BPYVqnEF9GQV5N3/VhFUCdsEwAIOQC0KfZ5LNBSRg=";
93 })
94
95 # remove after the 10.0 release
96 (fetchpatch {
97 name = "ibis-framework-arrow-18.patch";
98 url = "https://github.com/ibis-project/ibis/commit/5dc549b22c2eca29a11a31fb29deef7c1466a204.patch";
99 hash = "sha256-4i/g2uixdlkbE6x659wzZJ91FZpzwOVkF6ZeXkiCP3I=";
100 excludes = [
101 "poetry.lock"
102 "requirements-dev.txt"
103 ];
104 })
105 ];
106
107 build-system = [
108 poetry-core
109 poetry-dynamic-versioning
110 ];
111
112 dontBypassPoetryDynamicVersioning = true;
113 env.POETRY_DYNAMIC_VERSIONING_BYPASS = lib.head (lib.strings.splitString "-" version);
114
115 dependencies = [
116 atpublic
117 parsy
118 python-dateutil
119 pytz
120 sqlglot
121 toolz
122 typing-extensions
123 ];
124
125 nativeCheckInputs = [
126 pytestCheckHook
127 black
128 filelock
129 hypothesis
130 pytest-benchmark
131 pytest-httpserver
132 pytest-mock
133 pytest-randomly
134 pytest-snapshot
135 pytest-timeout
136 # this dependency is still needed due to use of strict markers and
137 # `pytest.mark.xdist_group` in the ibis codebase
138 pytest-xdist
139 ] ++ lib.concatMap (name: optional-dependencies.${name}) testBackends;
140
141 pytestFlagsArray = [
142 "-m"
143 "'${lib.concatStringsSep " or " testBackends} or core'"
144 ];
145
146 disabledTests = [
147 # tries to download duckdb extensions
148 "test_attach_sqlite"
149 "test_connect_extensions"
150 "test_load_extension"
151 "test_read_sqlite"
152 "test_register_sqlite"
153 # requires network connection
154 "test_s3_403_fallback"
155 "test_hugging_face"
156 # requires pytest 8.2+
157 "test_roundtrip_delta"
158 ];
159
160 # patch out tests that check formatting with black
161 postPatch = ''
162 find ibis/tests -type f -name '*.py' -exec sed -i \
163 -e '/^ *assert_decompile_roundtrip/d' \
164 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
165 '';
166
167 preCheck = ''
168 HOME="$TMPDIR"
169 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
170
171 # copy the test data to a directory
172 ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
173 '';
174
175 postCheck = ''
176 rm -r "$IBIS_TEST_DATA_DIRECTORY"
177 '';
178
179 pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
180
181 optional-dependencies = {
182 bigquery = [
183 db-dtypes
184 google-cloud-bigquery
185 google-cloud-bigquery-storage
186 pyarrow
187 pyarrow-hotfix
188 pydata-google-auth
189 numpy
190 pandas
191 rich
192 ];
193 clickhouse = [
194 clickhouse-connect
195 pyarrow
196 pyarrow-hotfix
197 numpy
198 pandas
199 rich
200 ];
201 datafusion = [
202 datafusion
203 pyarrow
204 pyarrow-hotfix
205 numpy
206 pandas
207 rich
208 ];
209 druid = [
210 pydruid
211 pyarrow
212 pyarrow-hotfix
213 numpy
214 pandas
215 rich
216 ];
217 duckdb = [
218 duckdb
219 pyarrow
220 pyarrow-hotfix
221 numpy
222 packaging
223 pandas
224 rich
225 ];
226 flink = [
227 pyarrow
228 pyarrow-hotfix
229 numpy
230 pandas
231 rich
232 ];
233 geospatial = [
234 geopandas
235 shapely
236 ];
237 mssql = [
238 pyodbc
239 pyarrow
240 pyarrow-hotfix
241 numpy
242 pandas
243 rich
244 ];
245 mysql = [
246 pymysql
247 pyarrow
248 pyarrow-hotfix
249 numpy
250 pandas
251 rich
252 ];
253 oracle = [
254 oracledb
255 packaging
256 pyarrow
257 pyarrow-hotfix
258 numpy
259 pandas
260 rich
261 ];
262 polars = [
263 polars
264 packaging
265 pyarrow
266 pyarrow-hotfix
267 numpy
268 pandas
269 rich
270 ];
271 postgres = [
272 psycopg2
273 pyarrow
274 pyarrow-hotfix
275 numpy
276 pandas
277 rich
278 ];
279 pyspark = [
280 pyspark
281 packaging
282 pyarrow
283 pyarrow-hotfix
284 numpy
285 pandas
286 rich
287 ];
288 snowflake = [
289 snowflake-connector-python
290 pyarrow
291 pyarrow-hotfix
292 numpy
293 pandas
294 rich
295 ];
296 sqlite = [
297 regex
298 pyarrow
299 pyarrow-hotfix
300 numpy
301 pandas
302 rich
303 ];
304 trino = [
305 trino-python-client
306 pyarrow
307 pyarrow-hotfix
308 numpy
309 pandas
310 rich
311 ];
312 visualization = [ graphviz ];
313 decompiler = [ black ];
314 examples = [ pins ] ++ pins.optional-dependencies.gcs;
315 };
316
317 meta = with lib; {
318 description = "Productivity-centric Python Big Data Framework";
319 homepage = "https://github.com/ibis-project/ibis";
320 changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
321 license = licenses.asl20;
322 maintainers = with maintainers; [ cpcloud ];
323 };
324}