pyrox.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5 pythonOlder,
6 pytestCheckHook,
7 atpublic,
8 bidict,
9 black,
10 clickhouse-connect,
11 dask,
12 datafusion,
13 db-dtypes,
14 duckdb,
15 duckdb-engine,
16 filelock,
17 geoalchemy2,
18 geopandas,
19 google-cloud-bigquery,
20 google-cloud-bigquery-storage,
21 graphviz,
22 hypothesis,
23 multipledispatch,
24 numpy,
25 oracledb,
26 packaging,
27 pandas,
28 parsy,
29 pins,
30 poetry-core,
31 poetry-dynamic-versioning,
32 polars,
33 psycopg2,
34 pyarrow,
35 pyarrow-hotfix,
36 pydata-google-auth,
37 pydruid,
38 pymysql,
39 pyodbc,
40 pyspark,
41 pytest-benchmark,
42 pytest-httpserver,
43 pytest-mock,
44 pytest-randomly,
45 pytest-snapshot,
46 pytest-timeout,
47 pytest-xdist,
48 python-dateutil,
49 pytz,
50 regex,
51 rich,
52 shapely,
53 snowflake-connector-python,
54 snowflake-sqlalchemy,
55 sqlalchemy,
56 sqlalchemy-views,
57 sqlglot,
58 sqlite,
59 toolz,
60 trino-python-client,
61 typing-extensions,
62}:
63let
64 testBackends = [
65 "datafusion"
66 "duckdb"
67 "pandas"
68 "sqlite"
69 ];
70
71 ibisTestingData = fetchFromGitHub {
72 name = "ibis-testing-data";
73 owner = "ibis-project";
74 repo = "testing-data";
75 # https://github.com/ibis-project/ibis/blob/9.0.0/nix/overlay.nix#L20-L26
76 rev = "1922bd4617546b877e66e78bb2b87abeb510cf8e";
77 hash = "sha256-l5d7r/6Voy6N2pXq3IivLX3N0tNfKKwsbZXRexzc8Z8=";
78 };
79in
80
81buildPythonPackage rec {
82 pname = "ibis-framework";
83 version = "9.0.0";
84 pyproject = true;
85
86 disabled = pythonOlder "3.9";
87
88 src = fetchFromGitHub {
89 name = "ibis-source";
90 repo = "ibis";
91 owner = "ibis-project";
92 rev = "refs/tags/${version}";
93 hash = "sha256-ebTYCBL1zm2Rmwg998x2kYvKhyQDk8Di1pcx5lR37xo=";
94 };
95
96 nativeBuildInputs = [
97 poetry-core
98 poetry-dynamic-versioning
99 ];
100
101 POETRY_DYNAMIC_VERSIONING_BYPASS = version;
102
103 propagatedBuildInputs = [
104 atpublic
105 bidict
106 multipledispatch
107 numpy
108 pandas
109 parsy
110 pyarrow
111 pyarrow-hotfix
112 python-dateutil
113 pytz
114 rich
115 sqlglot
116 toolz
117 typing-extensions
118 ];
119
120 nativeCheckInputs = [
121 pytestCheckHook
122 black
123 filelock
124 hypothesis
125 pytest-benchmark
126 pytest-httpserver
127 pytest-mock
128 pytest-randomly
129 pytest-snapshot
130 pytest-timeout
131 pytest-xdist
132 ] ++ lib.concatMap (name: passthru.optional-dependencies.${name}) testBackends;
133
134 pytestFlagsArray = [
135 "--dist=loadgroup"
136 "-m"
137 "'${lib.concatStringsSep " or " testBackends} or core'"
138 ];
139
140 disabledTests = [
141 # breakage from sqlalchemy2 truediv changes
142 "test_tpc_h17"
143 # tries to download duckdb extensions
144 "test_attach_sqlite"
145 "test_connect_extensions"
146 "test_load_extension"
147 "test_read_sqlite"
148 "test_register_sqlite"
149 # duckdb does not respect sample_size=2 (reads 3 lines of csv).
150 "test_csv_reregister_schema"
151 # duckdb fails with:
152 # "This function can not be called with an active transaction!, commit or abort the existing one first"
153 "test_vectorized_udf"
154 "test_s3_403_fallback"
155 "test_map_merge_udf"
156 "test_udf"
157 "test_map_udf"
158 # DataFusion error
159 "datafusion"
160 # pluggy.PluggyTeardownRaisedWarning
161 "test_repr_png_is_not_none_in_not_interactive"
162 "test_interval_arithmetic"
163 ];
164
165 # patch out tests that check formatting with black
166 postPatch = ''
167 find ibis/tests -type f -name '*.py' -exec sed -i \
168 -e '/^ *assert_decompile_roundtrip/d' \
169 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
170 '';
171
172 preCheck = ''
173 HOME="$TMPDIR"
174 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
175
176 # copy the test data to a directory
177 ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
178 '';
179
180 postCheck = ''
181 rm -r "$IBIS_TEST_DATA_DIRECTORY"
182 '';
183
184 pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
185
186 passthru = {
187 optional-dependencies = {
188 bigquery = [
189 db-dtypes
190 google-cloud-bigquery
191 google-cloud-bigquery-storage
192 pydata-google-auth
193 ];
194 clickhouse = [
195 clickhouse-connect
196 sqlalchemy
197 ];
198 dask = [
199 dask
200 regex
201 ];
202 datafusion = [ datafusion ];
203 druid = [
204 pydruid
205 sqlalchemy
206 ];
207 duckdb = [
208 duckdb
209 duckdb-engine
210 sqlalchemy
211 sqlalchemy-views
212 ];
213 flink = [ ];
214 geospatial = [
215 geoalchemy2
216 geopandas
217 shapely
218 ];
219 mssql = [
220 sqlalchemy
221 pyodbc
222 sqlalchemy-views
223 ];
224 mysql = [
225 sqlalchemy
226 pymysql
227 sqlalchemy-views
228 ];
229 oracle = [
230 sqlalchemy
231 oracledb
232 packaging
233 sqlalchemy-views
234 ];
235 pandas = [ regex ];
236 polars = [
237 polars
238 packaging
239 ];
240 postgres = [
241 psycopg2
242 sqlalchemy
243 sqlalchemy-views
244 ];
245 pyspark = [
246 pyspark
247 sqlalchemy
248 packaging
249 ];
250 snowflake = [
251 snowflake-connector-python
252 snowflake-sqlalchemy
253 sqlalchemy-views
254 packaging
255 ];
256 sqlite = [
257 regex
258 sqlalchemy
259 sqlalchemy-views
260 ];
261 trino = [
262 trino-python-client
263 sqlalchemy
264 sqlalchemy-views
265 ];
266 visualization = [ graphviz ];
267 decompiler = [ black ];
268 examples = [ pins ] ++ pins.optional-dependencies.gcs;
269 };
270 };
271
272 meta = with lib; {
273 description = "Productivity-centric Python Big Data Framework";
274 homepage = "https://github.com/ibis-project/ibis";
275 changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
276 license = licenses.asl20;
277 maintainers = with maintainers; [ cpcloud ];
278 };
279}