tjh.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5
6 # build-system
7 hatchling,
8
9 # dependencies
10 atpublic,
11 parsy,
12 python-dateutil,
13 sqlglot,
14 toolz,
15 typing-extensions,
16 tzdata,
17
18 # tests
19 pytestCheckHook,
20 black,
21 filelock,
22 hypothesis,
23 pytest-benchmark,
24 pytest-httpserver,
25 pytest-mock,
26 pytest-randomly,
27 pytest-snapshot,
28 pytest-timeout,
29 pytest-xdist,
30 writableTmpDirAsHomeHook,
31
32 # optional-dependencies
33 # - athena
34 pyathena,
35 fsspec,
36 # - bigquery
37 db-dtypes,
38 google-cloud-bigquery,
39 google-cloud-bigquery-storage,
40 pyarrow,
41 pyarrow-hotfix,
42 pydata-google-auth,
43 numpy,
44 pandas,
45 rich,
46 # - clickhouse
47 clickhouse-connect,
48 # - databricks
49 # databricks-sql-connector-core, (unpackaged)
50 # - datafusion
51 datafusion,
52 # - druid
53 pydruid,
54 # - duckdb
55 duckdb,
56 packaging,
57 # - flink
58 # - geospatial
59 geopandas,
60 shapely,
61 # - mssql
62 pyodbc,
63 # - mysql
64 pymysql,
65 # - oracle
66 oracledb,
67 # - polars
68 polars,
69 # - postgres
70 psycopg2,
71 # - pyspark
72 pyspark,
73 # - snowflake
74 snowflake-connector-python,
75 # sqlite
76 regex,
77 # - trino
78 trino-python-client,
79 # - visualization
80 graphviz,
81 # examples
82 pins,
83}:
84let
85 testBackends = [
86 "duckdb"
87 "sqlite"
88 ];
89
90 ibisTestingData = fetchFromGitHub {
91 owner = "ibis-project";
92 repo = "testing-data";
93 # https://github.com/ibis-project/ibis/blob/10.5.0/nix/overlay.nix#L94-L100
94 rev = "b26bd40cf29004372319df620c4bbe41420bb6f8";
95 hash = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU=";
96 };
97in
98
99buildPythonPackage rec {
100 pname = "ibis-framework";
101 version = "10.5.0";
102 pyproject = true;
103
104 src = fetchFromGitHub {
105 owner = "ibis-project";
106 repo = "ibis";
107 tag = version;
108 hash = "sha256-KJPl5bkD/tQlHY2k0b9zok5YCPekaXw7Y9z8P4AD3FQ=";
109 };
110
111 build-system = [
112 hatchling
113 ];
114
115 dependencies = [
116 atpublic
117 parsy
118 python-dateutil
119 sqlglot
120 toolz
121 typing-extensions
122 tzdata
123 ];
124
125 nativeCheckInputs = [
126 pytestCheckHook
127 black
128 filelock
129 hypothesis
130 pytest-benchmark
131 pytest-httpserver
132 pytest-mock
133 pytest-randomly
134 pytest-snapshot
135 pytest-timeout
136 # this dependency is still needed due to use of strict markers and
137 # `pytest.mark.xdist_group` in the ibis codebase
138 pytest-xdist
139 writableTmpDirAsHomeHook
140 ]
141 ++ lib.concatMap (name: optional-dependencies.${name}) testBackends;
142
143 pytestFlags = [
144 "--benchmark-disable"
145 ];
146
147 enabledTestMarks = testBackends ++ [ "core" ];
148
149 disabledTests = [
150 # tries to download duckdb extensions
151 "test_attach_sqlite"
152 "test_connect_extensions"
153 "test_load_extension"
154 "test_read_csv_with_types"
155 "test_read_sqlite"
156 "test_register_sqlite"
157 "test_roundtrip_xlsx"
158
159 # AssertionError: value does not match the expected value in snapshot
160 "test_union_aliasing"
161
162 # requires network connection
163 "test_s3_403_fallback"
164 "test_hugging_face"
165
166 # requires pytest 8.2+
167 "test_roundtrip_delta"
168
169 # AssertionError: value does not match the expected value in snapshot ibis/backends/tests/snapshots/test_sql/test_rewrite_context/sqlite/out.sql
170 "test_rewrite_context"
171
172 # Assertion error comparing a calculated version string with the actual (during nixpkgs-review)
173 "test_builtin_scalar_noargs"
174
175 # duckdb ParserError: syntax error at or near "AT"
176 "test_90"
177 ];
178
179 # patch out tests that check formatting with black
180 postPatch = ''
181 find ibis/tests -type f -name '*.py' -exec sed -i \
182 -e '/^ *assert_decompile_roundtrip/d' \
183 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
184 '';
185
186 preCheck = ''
187 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
188
189 # copy the test data to a directory
190 ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
191 '';
192
193 postCheck = ''
194 rm -r "$IBIS_TEST_DATA_DIRECTORY"
195 '';
196
197 pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
198
199 optional-dependencies = {
200 athena = [
201 pyathena
202 pyarrow
203 pyarrow-hotfix
204 numpy
205 pandas
206 rich
207 packaging
208 fsspec
209 ];
210 bigquery = [
211 db-dtypes
212 google-cloud-bigquery
213 google-cloud-bigquery-storage
214 pyarrow
215 pyarrow-hotfix
216 pydata-google-auth
217 numpy
218 pandas
219 rich
220 ];
221 clickhouse = [
222 clickhouse-connect
223 pyarrow
224 pyarrow-hotfix
225 numpy
226 pandas
227 rich
228 ];
229 databricks = [
230 # databricks-sql-connector-core (unpackaged)
231 pyarrow
232 pyarrow-hotfix
233 numpy
234 pandas
235 rich
236 ];
237 datafusion = [
238 datafusion
239 pyarrow
240 pyarrow-hotfix
241 numpy
242 pandas
243 rich
244 ];
245 druid = [
246 pydruid
247 pyarrow
248 pyarrow-hotfix
249 numpy
250 pandas
251 rich
252 ];
253 duckdb = [
254 duckdb
255 pyarrow
256 pyarrow-hotfix
257 numpy
258 pandas
259 rich
260 packaging
261 ];
262 flink = [
263 pyarrow
264 pyarrow-hotfix
265 numpy
266 pandas
267 rich
268 ];
269 geospatial = [
270 geopandas
271 shapely
272 ];
273 mssql = [
274 pyodbc
275 pyarrow
276 pyarrow-hotfix
277 numpy
278 pandas
279 rich
280 ];
281 mysql = [
282 pymysql
283 pyarrow
284 pyarrow-hotfix
285 numpy
286 pandas
287 rich
288 ];
289 oracle = [
290 oracledb
291 packaging
292 pyarrow
293 pyarrow-hotfix
294 numpy
295 pandas
296 rich
297 ];
298 polars = [
299 polars
300 packaging
301 pyarrow
302 pyarrow-hotfix
303 numpy
304 pandas
305 rich
306 ];
307 postgres = [
308 psycopg2
309 pyarrow
310 pyarrow-hotfix
311 numpy
312 pandas
313 rich
314 ];
315 pyspark = [
316 pyspark
317 packaging
318 pyarrow
319 pyarrow-hotfix
320 numpy
321 pandas
322 rich
323 ];
324 snowflake = [
325 snowflake-connector-python
326 pyarrow
327 pyarrow-hotfix
328 numpy
329 pandas
330 rich
331 ];
332 sqlite = [
333 regex
334 pyarrow
335 pyarrow-hotfix
336 numpy
337 pandas
338 rich
339 ];
340 trino = [
341 trino-python-client
342 pyarrow
343 pyarrow-hotfix
344 numpy
345 pandas
346 rich
347 ];
348 visualization = [ graphviz ];
349 decompiler = [ black ];
350 examples = [ pins ] ++ pins.optional-dependencies.gcs;
351 };
352
353 meta = {
354 description = "Productivity-centric Python Big Data Framework";
355 homepage = "https://github.com/ibis-project/ibis";
356 changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
357 license = lib.licenses.asl20;
358 maintainers = with lib.maintainers; [
359 cpcloud
360 sarahec
361 ];
362 };
363}