pyrox.dev
1{ lib
2, buildPythonPackage
3, fetchFromGitHub
4, pythonOlder
5, pytestCheckHook
6, atpublic
7, bidict
8, black
9, clickhouse-connect
10, dask
11, datafusion
12, db-dtypes
13, duckdb
14, duckdb-engine
15, filelock
16, geoalchemy2
17, geopandas
18, google-cloud-bigquery
19, google-cloud-bigquery-storage
20, graphviz-nox
21, hypothesis
22, multipledispatch
23, numpy
24, oracledb
25, packaging
26, pandas
27, parsy
28, poetry-core
29, poetry-dynamic-versioning
30, polars
31, pooch
32, psycopg2
33, pyarrow
34, pydata-google-auth
35, pydruid
36, pymysql
37, pyspark
38, pytest-benchmark
39, pytest-httpserver
40, pytest-mock
41, pytest-randomly
42, pytest-snapshot
43, pytest-xdist
44, python-dateutil
45, pytz
46, regex
47, rich
48, shapely
49, snowflake-connector-python
50, snowflake-sqlalchemy
51, sqlalchemy
52, sqlalchemy-views
53, sqlglot
54, sqlite
55, toolz
56, trino-python-client
57, typing-extensions
58}:
59let
60 testBackends = [ "datafusion" "duckdb" "pandas" "sqlite" ];
61
62 ibisTestingData = fetchFromGitHub {
63 name = "ibis-testing-data";
64 owner = "ibis-project";
65 repo = "testing-data";
66 rev = "2b3968deaa1a28791b2901dbbcc9bfd3d2f23e9b";
67 hash = "sha256-q1b5IcOl5oIFXP7/P5RufncjHEVrWp4NjoU2uo/BE9U=";
68 };
69in
70
71buildPythonPackage rec {
72 pname = "ibis-framework";
73 version = "6.1.0";
74 format = "pyproject";
75
76 disabled = pythonOlder "3.9";
77
78 src = fetchFromGitHub {
79 name = "ibis-source";
80 repo = "ibis";
81 owner = "ibis-project";
82 rev = "refs/tags/${version}";
83 hash = "sha256-+AtXgRNxPryP/fd/GQlLNxWbP6ozikqG2yBCp3dE0tY=";
84 };
85
86 nativeBuildInputs = [
87 poetry-core
88 poetry-dynamic-versioning
89 ];
90
91 POETRY_DYNAMIC_VERSIONING_BYPASS = version;
92
93 propagatedBuildInputs = [
94 atpublic
95 bidict
96 filelock
97 multipledispatch
98 numpy
99 pandas
100 parsy
101 pooch
102 pyarrow
103 python-dateutil
104 pytz
105 rich
106 sqlglot
107 toolz
108 typing-extensions
109 ]
110 ++ pooch.optional-dependencies.progress
111 ++ pooch.optional-dependencies.xxhash;
112
113 nativeCheckInputs = [
114 pytestCheckHook
115 hypothesis
116 pytest-benchmark
117 pytest-httpserver
118 pytest-mock
119 pytest-randomly
120 pytest-snapshot
121 pytest-xdist
122 ] ++ lib.concatMap (name: passthru.optional-dependencies.${name}) testBackends;
123
124 pytestFlagsArray = [
125 "--dist=loadgroup"
126 "-m"
127 "'${lib.concatStringsSep " or " testBackends} or core'"
128 # breakage from sqlalchemy2 truediv changes
129 "--deselect=ibis/tests/sql/test_sqlalchemy.py::test_tpc_h17"
130 # tries to download duckdb extensions
131 "--deselect=ibis/backends/duckdb/tests/test_register.py::test_register_sqlite"
132 "--deselect=ibis/backends/duckdb/tests/test_register.py::test_read_sqlite"
133
134 # duckdb does not respect sample_size=2 (reads 3 lines of csv).
135 "--deselect=ibis/backends/tests/test_register.py::test_csv_reregister_schema"
136
137 # duckdb fails with:
138 # "This function can not be called with an active transaction!, commit or abort the existing one first"
139 "--deselect=ibis/backends/tests/test_udf.py::test_vectorized_udf"
140 "--deselect=ibis/backends/tests/test_udf.py::test_map_merge_udf"
141 "--deselect=ibis/backends/tests/test_udf.py::test_udf"
142 "--deselect=ibis/backends/tests/test_udf.py::test_map_udf"
143
144 # pyarrow13 is not supported yet.
145 "--deselect=ibis/backends/tests/test_temporal.py::test_date_truncate"
146 "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp"
147 "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp"
148 "--deselect=ibis/backends/tests/test_temporal.py::test_interval_add_cast_column"
149 "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp"
150 "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp"
151 "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp"
152 "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp"
153 "--deselect=ibis/backends/tests/test_timecontext.py::test_context_adjustment_filter_before_window"
154 "--deselect=ibis/backends/tests/test_timecontext.py::test_context_adjustment_window_udf"
155 "--deselect=ibis/backends/tests/test_timecontext.py::test_context_adjustment_window_udf"
156 "--deselect=ibis/backends/tests/test_aggregation.py::test_aggregate_grouped"
157 ];
158
159 # patch out tests that check formatting with black
160 postPatch = ''
161 find ibis/tests -type f -name '*.py' -exec sed -i \
162 -e '/^ *assert_decompile_roundtrip/d' \
163 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
164 substituteInPlace pyproject.toml --replace 'sqlglot = ">=10.4.3,<12"' 'sqlglot = "*"'
165 '';
166
167 preCheck = ''
168 HOME="$TMPDIR"
169 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
170
171 # copy the test data to a directory
172 ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
173 '';
174
175 postCheck = ''
176 rm -r "$IBIS_TEST_DATA_DIRECTORY"
177 '';
178
179 pythonImportsCheck = [
180 "ibis"
181 ] ++ map (backend: "ibis.backends.${backend}") testBackends;
182
183 passthru = {
184 optional-dependencies = {
185 bigquery = [ db-dtypes google-cloud-bigquery google-cloud-bigquery-storage pydata-google-auth ];
186 clickhouse = [ clickhouse-connect sqlalchemy ];
187 dask = [ dask regex ];
188 datafusion = [ datafusion ];
189 druid = [ pydruid sqlalchemy ];
190 duckdb = [ duckdb duckdb-engine packaging sqlalchemy sqlalchemy-views ];
191 flink = [ ];
192 geospatial = [ geoalchemy2 geopandas shapely ];
193 mysql = [ sqlalchemy pymysql sqlalchemy-views ];
194 oracle = [ sqlalchemy oracledb packaging sqlalchemy-views ];
195 pandas = [ regex ];
196 polars = [ polars ];
197 postgres = [ psycopg2 sqlalchemy sqlalchemy-views ];
198 pyspark = [ pyspark sqlalchemy ];
199 snowflake = [ snowflake-connector-python snowflake-sqlalchemy sqlalchemy-views ];
200 sqlite = [ regex sqlalchemy sqlite sqlalchemy-views ];
201 trino = [ trino-python-client sqlalchemy sqlalchemy-views ];
202 visualization = [ graphviz-nox ];
203 decompiler = [ black ];
204 };
205 };
206
207 meta = with lib; {
208 description = "Productivity-centric Python Big Data Framework";
209 homepage = "https://github.com/ibis-project/ibis";
210 changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
211 license = licenses.asl20;
212 maintainers = with maintainers; [ cpcloud ];
213 };
214}