pyrox.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5
6 # build-system
7 hatchling,
8
9 # dependencies
10 atpublic,
11 parsy,
12 python-dateutil,
13 sqlglot,
14 toolz,
15 typing-extensions,
16 tzdata,
17
18 # tests
19 pytestCheckHook,
20 black,
21 filelock,
22 hypothesis,
23 pytest-benchmark,
24 pytest-httpserver,
25 pytest-mock,
26 pytest-randomly,
27 pytest-snapshot,
28 pytest-timeout,
29 pytest-xdist,
30 writableTmpDirAsHomeHook,
31
32 # optional-dependencies
33 # - athena
34 pyathena,
35 fsspec,
36 # - bigquery
37 db-dtypes,
38 google-cloud-bigquery,
39 google-cloud-bigquery-storage,
40 pyarrow,
41 pyarrow-hotfix,
42 pydata-google-auth,
43 numpy,
44 pandas,
45 rich,
46 # - clickhouse
47 clickhouse-connect,
48 # - databricks
49 # databricks-sql-connector-core, (unpackaged)
50 # - datafusion
51 datafusion,
52 # - druid
53 pydruid,
54 # - duckdb
55 duckdb,
56 packaging,
57 # - flink
58 # - geospatial
59 geopandas,
60 shapely,
61 # - mssql
62 pyodbc,
63 # - mysql
64 pymysql,
65 # - oracle
66 oracledb,
67 # - polars
68 polars,
69 # - postgres
70 psycopg2,
71 # - pyspark
72 pyspark,
73 # - snowflake
74 snowflake-connector-python,
75 # sqlite
76 regex,
77 # - trino
78 trino-python-client,
79 # - visualization
80 graphviz,
81 # examples
82 pins,
83}:
84let
85 testBackends = [
86 "duckdb"
87 "sqlite"
88 ];
89
90 ibisTestingData = fetchFromGitHub {
91 owner = "ibis-project";
92 repo = "testing-data";
93 # https://github.com/ibis-project/ibis/blob/10.5.0/nix/overlay.nix#L94-L100
94 rev = "b26bd40cf29004372319df620c4bbe41420bb6f8";
95 hash = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU=";
96 };
97in
98
99buildPythonPackage rec {
100 pname = "ibis-framework";
101 version = "10.5.0";
102 pyproject = true;
103
104 src = fetchFromGitHub {
105 owner = "ibis-project";
106 repo = "ibis";
107 tag = version;
108 hash = "sha256-KJPl5bkD/tQlHY2k0b9zok5YCPekaXw7Y9z8P4AD3FQ=";
109 };
110
111 build-system = [
112 hatchling
113 ];
114
115 pythonRelaxDeps = [
116 # "toolz"
117 ];
118
119 dependencies = [
120 atpublic
121 parsy
122 python-dateutil
123 sqlglot
124 toolz
125 typing-extensions
126 tzdata
127 ];
128
129 nativeCheckInputs = [
130 pytestCheckHook
131 black
132 filelock
133 hypothesis
134 pytest-benchmark
135 pytest-httpserver
136 pytest-mock
137 pytest-randomly
138 pytest-snapshot
139 pytest-timeout
140 # this dependency is still needed due to use of strict markers and
141 # `pytest.mark.xdist_group` in the ibis codebase
142 pytest-xdist
143 writableTmpDirAsHomeHook
144 ] ++ lib.concatMap (name: optional-dependencies.${name}) testBackends;
145
146 pytestFlagsArray = [
147 "-m"
148 "'${lib.concatStringsSep " or " testBackends} or core'"
149 ];
150
151 disabledTests = [
152 # tries to download duckdb extensions
153 "test_attach_sqlite"
154 "test_connect_extensions"
155 "test_load_extension"
156 "test_read_csv_with_types"
157 "test_read_sqlite"
158 "test_register_sqlite"
159 "test_roundtrip_xlsx"
160
161 # AssertionError: value does not match the expected value in snapshot
162 "test_union_aliasing"
163
164 # requires network connection
165 "test_s3_403_fallback"
166 "test_hugging_face"
167
168 # requires pytest 8.2+
169 "test_roundtrip_delta"
170
171 # AssertionError: value does not match the expected value in snapshot ibis/backends/tests/snapshots/test_sql/test_rewrite_context/sqlite/out.sql
172 "test_rewrite_context"
173 ];
174
175 # patch out tests that check formatting with black
176 postPatch = ''
177 find ibis/tests -type f -name '*.py' -exec sed -i \
178 -e '/^ *assert_decompile_roundtrip/d' \
179 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
180 '';
181
182 preCheck = ''
183 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
184
185 # copy the test data to a directory
186 ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
187 '';
188
189 postCheck = ''
190 rm -r "$IBIS_TEST_DATA_DIRECTORY"
191 '';
192
193 pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
194
195 optional-dependencies = {
196 athena = [
197 pyathena
198 pyarrow
199 pyarrow-hotfix
200 numpy
201 pandas
202 rich
203 packaging
204 fsspec
205 ];
206 bigquery = [
207 db-dtypes
208 google-cloud-bigquery
209 google-cloud-bigquery-storage
210 pyarrow
211 pyarrow-hotfix
212 pydata-google-auth
213 numpy
214 pandas
215 rich
216 ];
217 clickhouse = [
218 clickhouse-connect
219 pyarrow
220 pyarrow-hotfix
221 numpy
222 pandas
223 rich
224 ];
225 databricks = [
226 # databricks-sql-connector-core (unpackaged)
227 pyarrow
228 pyarrow-hotfix
229 numpy
230 pandas
231 rich
232 ];
233 datafusion = [
234 datafusion
235 pyarrow
236 pyarrow-hotfix
237 numpy
238 pandas
239 rich
240 ];
241 druid = [
242 pydruid
243 pyarrow
244 pyarrow-hotfix
245 numpy
246 pandas
247 rich
248 ];
249 duckdb = [
250 duckdb
251 pyarrow
252 pyarrow-hotfix
253 numpy
254 pandas
255 rich
256 packaging
257 ];
258 flink = [
259 pyarrow
260 pyarrow-hotfix
261 numpy
262 pandas
263 rich
264 ];
265 geospatial = [
266 geopandas
267 shapely
268 ];
269 mssql = [
270 pyodbc
271 pyarrow
272 pyarrow-hotfix
273 numpy
274 pandas
275 rich
276 ];
277 mysql = [
278 pymysql
279 pyarrow
280 pyarrow-hotfix
281 numpy
282 pandas
283 rich
284 ];
285 oracle = [
286 oracledb
287 packaging
288 pyarrow
289 pyarrow-hotfix
290 numpy
291 pandas
292 rich
293 ];
294 polars = [
295 polars
296 packaging
297 pyarrow
298 pyarrow-hotfix
299 numpy
300 pandas
301 rich
302 ];
303 postgres = [
304 psycopg2
305 pyarrow
306 pyarrow-hotfix
307 numpy
308 pandas
309 rich
310 ];
311 pyspark = [
312 pyspark
313 packaging
314 pyarrow
315 pyarrow-hotfix
316 numpy
317 pandas
318 rich
319 ];
320 snowflake = [
321 snowflake-connector-python
322 pyarrow
323 pyarrow-hotfix
324 numpy
325 pandas
326 rich
327 ];
328 sqlite = [
329 regex
330 pyarrow
331 pyarrow-hotfix
332 numpy
333 pandas
334 rich
335 ];
336 trino = [
337 trino-python-client
338 pyarrow
339 pyarrow-hotfix
340 numpy
341 pandas
342 rich
343 ];
344 visualization = [ graphviz ];
345 decompiler = [ black ];
346 examples = [ pins ] ++ pins.optional-dependencies.gcs;
347 };
348
349 meta = {
350 description = "Productivity-centric Python Big Data Framework";
351 homepage = "https://github.com/ibis-project/ibis";
352 changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
353 license = lib.licenses.asl20;
354 maintainers = with lib.maintainers; [ cpcloud ];
355 };
356}