tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5 pythonAtLeast,
6
7 # build-system
8 hatchling,
9
10 # dependencies
11 atpublic,
12 parsy,
13 python-dateutil,
14 sqlglot,
15 toolz,
16 typing-extensions,
17 tzdata,
18
19 # tests
20 pytestCheckHook,
21 black,
22 filelock,
23 hypothesis,
24 pytest-benchmark,
25 pytest-httpserver,
26 pytest-mock,
27 pytest-randomly,
28 pytest-snapshot,
29 pytest-timeout,
30 pytest-xdist,
31 writableTmpDirAsHomeHook,
32
33 # optional-dependencies
34 # - athena
35 pyathena,
36 fsspec,
37 # - bigquery
38 db-dtypes,
39 google-cloud-bigquery,
40 google-cloud-bigquery-storage,
41 pyarrow,
42 pyarrow-hotfix,
43 pydata-google-auth,
44 numpy,
45 pandas,
46 rich,
47 # - clickhouse
48 clickhouse-connect,
49 # - databricks
50 # databricks-sql-connector-core, (unpackaged)
51 # - datafusion
52 datafusion,
53 # - druid
54 pydruid,
55 # - duckdb
56 duckdb,
57 packaging,
58 # - flink
59 # - geospatial
60 geopandas,
61 shapely,
62 # - mssql
63 pyodbc,
64 # - mysql
65 pymysql,
66 # - oracle
67 oracledb,
68 # - polars
69 polars,
70 # - postgres
71 psycopg2,
72 # - pyspark
73 pyspark,
74 # - snowflake
75 snowflake-connector-python,
76 # sqlite
77 regex,
78 # - trino
79 trino-python-client,
80 # - visualization
81 graphviz,
82 # examples
83 pins,
84}:
85let
86 testBackends = [
87 "duckdb"
88 "sqlite"
89 ];
90
91 ibisTestingData = fetchFromGitHub {
92 owner = "ibis-project";
93 repo = "testing-data";
94 # https://github.com/ibis-project/ibis/blob/10.5.0/nix/overlay.nix#L94-L100
95 rev = "b26bd40cf29004372319df620c4bbe41420bb6f8";
96 hash = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU=";
97 };
98in
99
100buildPythonPackage (finalAttrs: {
101 pname = "ibis-framework";
102 version = "11.0.0";
103 pyproject = true;
104
105 src = fetchFromGitHub {
106 owner = "ibis-project";
107 repo = "ibis";
108 tag = finalAttrs.version;
109 hash = "sha256-hf5guWeX9WQbKaNrs7ALwwDxV1Rgeb5Z0PedTQ4P7S0=";
110 };
111
112 build-system = [
113 hatchling
114 ];
115
116 dependencies = [
117 atpublic
118 parsy
119 python-dateutil
120 sqlglot
121 toolz
122 typing-extensions
123 tzdata
124 ];
125
126 nativeCheckInputs = [
127 pytestCheckHook
128 black
129 filelock
130 hypothesis
131 pytest-benchmark
132 pytest-httpserver
133 pytest-mock
134 pytest-randomly
135 pytest-snapshot
136 pytest-timeout
137 # this dependency is still needed due to use of strict markers and
138 # `pytest.mark.xdist_group` in the ibis codebase
139 pytest-xdist
140 writableTmpDirAsHomeHook
141 ]
142 ++ lib.concatMap (name: finalAttrs.passthru.optional-dependencies.${name}) testBackends;
143
144 pytestFlags = [
145 "--benchmark-disable"
146 "-Wignore::FutureWarning"
147 ]
148 ++ lib.optionals (pythonAtLeast "3.14") [
149 # DeprecationWarning: '_UnionGenericAlias' is deprecated and slated for removal in Python 3.17
150 "-Wignore::DeprecationWarning"
151 # Multiple tests with warnings fail without it
152 "-Wignore::pytest.PytestUnraisableExceptionWarning"
153 ];
154
155 enabledTestMarks = testBackends ++ [ "core" ];
156
157 disabledTests = [
158 # tries to download duckdb extensions
159 "test_attach_sqlite"
160 "test_connect_extensions"
161 "test_load_extension"
162 "test_read_csv_with_types"
163 "test_read_sqlite"
164 "test_register_sqlite"
165 "test_roundtrip_xlsx"
166
167 # AssertionError: value does not match the expected value in snapshot
168 "test_union_aliasing"
169
170 # requires network connection
171 "test_s3_403_fallback"
172 "test_hugging_face"
173
174 # requires pytest 8.2+
175 "test_roundtrip_delta"
176
177 # AssertionError: value does not match the expected value in snapshot ibis/backends/tests/snapshots/test_sql/test_rewrite_context/sqlite/out.sql
178 "test_rewrite_context"
179
180 # Assertion error comparing a calculated version string with the actual (during nixpkgs-review)
181 "test_builtin_scalar_noargs"
182
183 # duckdb ParserError: syntax error at or near "AT"
184 "test_90"
185
186 # assert 0 == 3 (tests edge case behavior of databases)
187 "test_self_join_with_generated_keys"
188 ]
189 ++ lib.optionals (pythonAtLeast "3.14") [
190 # ExceptionGroup: multiple unraisable exception warnings (4 sub-exceptions)
191 "test_non_roundtripable_str_type"
192 "test_parse_dtype_roundtrip"
193
194 # AssertionError: value does not match the expected value in snapshot ...
195 "test_annotated_function_without_decoration"
196 "test_error_message"
197 "test_error_message_when_constructing_literal"
198 "test_signature_from_callable_with_keyword_only_arguments"
199 ];
200
201 # patch out tests that check formatting with black
202 postPatch = ''
203 find ibis/tests -type f -name '*.py' -exec sed -i \
204 -e '/^ *assert_decompile_roundtrip/d' \
205 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
206 '';
207
208 preCheck = ''
209 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
210
211 # copy the test data to a directory
212 ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
213 '';
214
215 postCheck = ''
216 rm -r "$IBIS_TEST_DATA_DIRECTORY"
217 '';
218
219 pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
220
221 optional-dependencies = {
222 athena = [
223 pyathena
224 pyarrow
225 pyarrow-hotfix
226 numpy
227 pandas
228 rich
229 packaging
230 fsspec
231 ];
232 bigquery = [
233 db-dtypes
234 google-cloud-bigquery
235 google-cloud-bigquery-storage
236 pyarrow
237 pyarrow-hotfix
238 pydata-google-auth
239 numpy
240 pandas
241 rich
242 ];
243 clickhouse = [
244 clickhouse-connect
245 pyarrow
246 pyarrow-hotfix
247 numpy
248 pandas
249 rich
250 ];
251 databricks = [
252 # databricks-sql-connector-core (unpackaged)
253 pyarrow
254 pyarrow-hotfix
255 numpy
256 pandas
257 rich
258 ];
259 datafusion = [
260 datafusion
261 pyarrow
262 pyarrow-hotfix
263 numpy
264 pandas
265 rich
266 ];
267 druid = [
268 pydruid
269 pyarrow
270 pyarrow-hotfix
271 numpy
272 pandas
273 rich
274 ];
275 duckdb = [
276 duckdb
277 pyarrow
278 pyarrow-hotfix
279 numpy
280 pandas
281 rich
282 packaging
283 ];
284 flink = [
285 pyarrow
286 pyarrow-hotfix
287 numpy
288 pandas
289 rich
290 ];
291 geospatial = [
292 geopandas
293 shapely
294 ];
295 mssql = [
296 pyodbc
297 pyarrow
298 pyarrow-hotfix
299 numpy
300 pandas
301 rich
302 ];
303 mysql = [
304 pymysql
305 pyarrow
306 pyarrow-hotfix
307 numpy
308 pandas
309 rich
310 ];
311 oracle = [
312 oracledb
313 packaging
314 pyarrow
315 pyarrow-hotfix
316 numpy
317 pandas
318 rich
319 ];
320 polars = [
321 polars
322 packaging
323 pyarrow
324 pyarrow-hotfix
325 numpy
326 pandas
327 rich
328 ];
329 postgres = [
330 psycopg2
331 pyarrow
332 pyarrow-hotfix
333 numpy
334 pandas
335 rich
336 ];
337 pyspark = [
338 pyspark
339 packaging
340 pyarrow
341 pyarrow-hotfix
342 numpy
343 pandas
344 rich
345 ];
346 snowflake = [
347 snowflake-connector-python
348 pyarrow
349 pyarrow-hotfix
350 numpy
351 pandas
352 rich
353 ];
354 sqlite = [
355 regex
356 pyarrow
357 pyarrow-hotfix
358 numpy
359 pandas
360 rich
361 ];
362 trino = [
363 trino-python-client
364 pyarrow
365 pyarrow-hotfix
366 numpy
367 pandas
368 rich
369 ];
370 visualization = [ graphviz ];
371 decompiler = [ black ];
372 examples = [ pins ] ++ pins.optional-dependencies.gcs;
373 };
374
375 meta = {
376 description = "Productivity-centric Python Big Data Framework";
377 homepage = "https://github.com/ibis-project/ibis";
378 changelog = "https://github.com/ibis-project/ibis/blob/${finalAttrs.src.tag}/docs/release_notes.md";
379 license = lib.licenses.asl20;
380 maintainers = with lib.maintainers; [
381 cpcloud
382 sarahec
383 ];
384 };
385})