tjh.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchFromGitHub,
5 pythonOlder,
6 pytestCheckHook,
7 atpublic,
8 bidict,
9 black,
10 clickhouse-connect,
11 dask,
12 datafusion,
13 db-dtypes,
14 duckdb,
15 filelock,
16 geopandas,
17 google-cloud-bigquery,
18 google-cloud-bigquery-storage,
19 graphviz,
20 hypothesis,
21 multipledispatch,
22 numpy,
23 oracledb,
24 packaging,
25 pandas,
26 parsy,
27 pins,
28 poetry-core,
29 poetry-dynamic-versioning,
30 polars,
31 psycopg2,
32 pyarrow,
33 pyarrow-hotfix,
34 pydata-google-auth,
35 pydruid,
36 pymysql,
37 pyodbc,
38 pyspark,
39 pytest-benchmark,
40 pytest-httpserver,
41 pytest-mock,
42 pytest-randomly,
43 pytest-snapshot,
44 pytest-timeout,
45 pytest-xdist,
46 python-dateutil,
47 pytz,
48 regex,
49 rich,
50 shapely,
51 snowflake-connector-python,
52 sqlglot,
53 sqlite,
54 toolz,
55 trino-python-client,
56 typing-extensions,
57}:
58let
59 testBackends = [
60 "datafusion"
61 "duckdb"
62 "pandas"
63 "sqlite"
64 ];
65
66 ibisTestingData = fetchFromGitHub {
67 name = "ibis-testing-data";
68 owner = "ibis-project";
69 repo = "testing-data";
70 # https://github.com/ibis-project/ibis/blob/9.1.0/nix/overlay.nix#L20-L26
71 rev = "6737d1cb5951cabaccd095a3ae62a93dbd11ecb9";
72 hash = "sha256-MoVTZPWh4KVlrICYACrgfeLdl/fqoa1iweNg3zUtdrs=";
73 };
74in
75
76buildPythonPackage rec {
77 pname = "ibis-framework";
78 version = "9.1.0";
79 pyproject = true;
80
81 disabled = pythonOlder "3.10";
82
83 src = fetchFromGitHub {
84 name = "ibis-source";
85 repo = "ibis";
86 owner = "ibis-project";
87 rev = "refs/tags/${version}";
88 hash = "sha256-GmzmXzYMs7K7B//is3ZoD4muPAkb0tM56zFBbsA+NEo=";
89 };
90
91 nativeBuildInputs = [
92 poetry-core
93 poetry-dynamic-versioning
94 ];
95
96 dontBypassPoetryDynamicVersioning = true;
97 env.POETRY_DYNAMIC_VERSIONING_BYPASS = lib.head (lib.strings.splitString "-" version);
98
99 propagatedBuildInputs = [
100 atpublic
101 bidict
102 multipledispatch
103 numpy
104 pandas
105 parsy
106 pyarrow
107 pyarrow-hotfix
108 python-dateutil
109 pytz
110 rich
111 sqlglot
112 toolz
113 typing-extensions
114 ];
115
116 nativeCheckInputs = [
117 pytestCheckHook
118 black
119 filelock
120 hypothesis
121 pytest-benchmark
122 pytest-httpserver
123 pytest-mock
124 pytest-randomly
125 pytest-snapshot
126 pytest-timeout
127 pytest-xdist
128 ] ++ lib.concatMap (name: passthru.optional-dependencies.${name}) testBackends;
129
130 pytestFlagsArray = [
131 "--dist=loadgroup"
132 "-m"
133 "'${lib.concatStringsSep " or " testBackends} or core'"
134 ];
135
136 disabledTests = [
137 # tries to download duckdb extensions
138 "test_attach_sqlite"
139 "test_connect_extensions"
140 "test_load_extension"
141 "test_read_sqlite"
142 "test_register_sqlite"
143 # requires network connection
144 "test_s3_403_fallback"
145 # requires pytest 8.2+
146 "test_roundtrip_delta"
147 ];
148
149 # patch out tests that check formatting with black
150 postPatch = ''
151 find ibis/tests -type f -name '*.py' -exec sed -i \
152 -e '/^ *assert_decompile_roundtrip/d' \
153 -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
154 '';
155
156 preCheck = ''
157 HOME="$TMPDIR"
158 export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
159
160 # copy the test data to a directory
161 ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
162 '';
163
164 postCheck = ''
165 rm -r "$IBIS_TEST_DATA_DIRECTORY"
166 '';
167
168 pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
169
170 passthru = {
171 optional-dependencies = {
172 bigquery = [
173 db-dtypes
174 google-cloud-bigquery
175 google-cloud-bigquery-storage
176 pydata-google-auth
177 ];
178 clickhouse = [ clickhouse-connect ];
179 dask = [
180 dask
181 regex
182 packaging
183 ];
184 datafusion = [ datafusion ];
185 druid = [ pydruid ];
186 duckdb = [ duckdb ];
187 flink = [ ];
188 geospatial = [
189 geopandas
190 shapely
191 ];
192 mssql = [ pyodbc ];
193 mysql = [ pymysql ];
194 oracle = [
195 oracledb
196 packaging
197 ];
198 pandas = [
199 regex
200 packaging
201 ];
202 polars = [
203 polars
204 packaging
205 ];
206 postgres = [ psycopg2 ];
207 pyspark = [
208 pyspark
209 packaging
210 ];
211 snowflake = [ snowflake-connector-python ];
212 sqlite = [ regex ];
213 trino = [ trino-python-client ];
214 visualization = [ graphviz ];
215 decompiler = [ black ];
216 examples = [ pins ] ++ pins.optional-dependencies.gcs;
217 };
218 };
219
220 meta = with lib; {
221 description = "Productivity-centric Python Big Data Framework";
222 homepage = "https://github.com/ibis-project/ibis";
223 changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
224 license = licenses.asl20;
225 maintainers = with maintainers; [ cpcloud ];
226 };
227}