tjh.dev
Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)
1{ stdenv
2, lib
3, fetchurl
4, fetchFromGitHub
5, fixDarwinDylibNames
6, abseil-cpp
7, autoconf
8, aws-sdk-cpp
9, boost
10, brotli
11, c-ares
12, cmake
13, crc32c
14, curl
15, flatbuffers
16, gflags
17, glog
18, google-cloud-cpp
19, grpc
20, gtest
21, jemalloc
22, libnsl
23, lz4
24, minio
25, nlohmann_json
26, openssl
27, perl
28, protobuf
29, python3
30, rapidjson
31, re2
32, snappy
33, thrift
34, tzdata
35, utf8proc
36, which
37, zlib
38, zstd
39, enableShared ? !stdenv.hostPlatform.isStatic
40, enableFlight ? !stdenv.isDarwin # libnsl is not supported on darwin
41, enableJemalloc ? !(stdenv.isAarch64 && stdenv.isDarwin)
42 # boost/process is broken in 1.69 on darwin, but fixed in 1.70 and
43 # non-existent in older versions
44 # see https://github.com/boostorg/process/issues/55
45, enableS3 ? (!stdenv.isDarwin) || (lib.versionOlder boost.version "1.69" || lib.versionAtLeast boost.version "1.70")
46, enableGcs ? !stdenv.isDarwin # google-cloud-cpp is not supported on darwin
47}:
48
49assert lib.asserts.assertMsg
50 ((enableS3 && stdenv.isDarwin) -> (lib.versionOlder boost.version "1.69" || lib.versionAtLeast boost.version "1.70"))
51 "S3 on Darwin requires Boost != 1.69";
52
53let
54 arrow-testing = fetchFromGitHub {
55 owner = "apache";
56 repo = "arrow-testing";
57 rev = "1d8525e109a12a8c67c489eba48715a199609153";
58 hash = "sha256-tesDW/1yRyhZtpLbPvCVEsocs6KtstYofxB5GiSMEFM=";
59 };
60
61 parquet-testing = fetchFromGitHub {
62 owner = "apache";
63 repo = "parquet-testing";
64 rev = "d4d485956a643c693b5549e1a62d52ca61c170f1";
65 hash = "sha256-GmOAS8gGhzDI0WzORMkWHRRUl/XBwmNen2d3VefZxxc=";
66 };
67
68in
69stdenv.mkDerivation rec {
70 pname = "arrow-cpp";
71 version = "6.0.1";
72
73 src = fetchurl {
74 url =
75 "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
76 hash = "sha256-N4az0t+VTQeLPmj5jS5a7Lqj+irM8HXXo6E8GHucUpQ=";
77 };
78 sourceRoot = "apache-arrow-${version}/cpp";
79
80 ${if enableJemalloc then "ARROW_JEMALLOC_URL" else null} = jemalloc.src;
81
82 ARROW_MIMALLOC_URL = fetchFromGitHub {
83 # From
84 # ./cpp/cmake_modules/ThirdpartyToolchain.cmake
85 # ./cpp/thirdparty/versions.txt
86 owner = "microsoft";
87 repo = "mimalloc";
88 rev = "v1.7.2";
89 hash = "sha256-yHupYFgC8mJuLUSpuEAfwF7l6Ue4EiuO1Q4qN4T6wWc=";
90 };
91
92 ARROW_XSIMD_URL = fetchFromGitHub {
93 owner = "xtensor-stack";
94 repo = "xsimd";
95 rev = "aeec9c872c8b475dedd7781336710f2dd2666cb2";
96 hash = "sha256-vWKdJkieKhaxyAJhijXUmD7NmNvMWd79PskQojulA1w=";
97 };
98
99 patches = [
100 # patch to fix python-test
101 ./darwin.patch
102 ];
103
104 nativeBuildInputs = [
105 cmake
106 autoconf # for vendored jemalloc
107 flatbuffers
108 ] ++ lib.optional stdenv.isDarwin fixDarwinDylibNames;
109 buildInputs = [
110 boost
111 brotli
112 flatbuffers
113 gflags
114 glog
115 gtest
116 lz4
117 rapidjson
118 re2
119 snappy
120 thrift
121 utf8proc
122 zlib
123 zstd
124 ] ++ lib.optionals enableShared [
125 python3.pkgs.python
126 python3.pkgs.numpy
127 ] ++ lib.optionals enableFlight [
128 grpc
129 libnsl
130 openssl
131 protobuf
132 ] ++ lib.optionals enableS3 [ aws-sdk-cpp openssl ]
133 ++ lib.optionals enableGcs [
134 abseil-cpp
135 crc32c
136 curl
137 google-cloud-cpp
138 nlohmann_json
139 ];
140
141 preConfigure = ''
142 patchShebangs build-support/
143 substituteInPlace "src/arrow/vendored/datetime/tz.cpp" \
144 --replace 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";'
145 '';
146
147 cmakeFlags = [
148 "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON"
149 "-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}"
150 "-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}"
151 "-DARROW_BUILD_TESTS=ON"
152 "-DARROW_VERBOSE_THIRDPARTY_BUILD=ON"
153 "-DARROW_DEPENDENCY_SOURCE=SYSTEM"
154 "-DThrift_SOURCE=AUTO" # search for Thrift using pkg-config (ThriftConfig.cmake requires OpenSSL and libevent)
155 "-DARROW_DEPENDENCY_USE_SHARED=${if enableShared then "ON" else "OFF"}"
156 "-DARROW_COMPUTE=ON"
157 "-DARROW_CSV=ON"
158 "-DARROW_DATASET=ON"
159 "-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}"
160 "-DARROW_JSON=ON"
161 "-DARROW_PLASMA=ON"
162 # Disable Python for static mode because openblas is currently broken there.
163 "-DARROW_PYTHON=${if enableShared then "ON" else "OFF"}"
164 "-DARROW_USE_GLOG=ON"
165 "-DARROW_WITH_BROTLI=ON"
166 "-DARROW_WITH_LZ4=ON"
167 "-DARROW_WITH_SNAPPY=ON"
168 "-DARROW_WITH_UTF8PROC=ON"
169 "-DARROW_WITH_ZLIB=ON"
170 "-DARROW_WITH_ZSTD=ON"
171 "-DARROW_MIMALLOC=ON"
172 # Parquet options:
173 "-DARROW_PARQUET=ON"
174 "-DPARQUET_BUILD_EXECUTABLES=ON"
175 "-DARROW_FLIGHT=${if enableFlight then "ON" else "OFF"}"
176 "-DARROW_S3=${if enableS3 then "ON" else "OFF"}"
177 "-DARROW_GCS=${if enableGcs then "ON" else "OFF"}"
178 ] ++ lib.optionals (!enableShared) [
179 "-DARROW_TEST_LINKAGE=static"
180 ] ++ lib.optionals stdenv.isDarwin [
181 "-DCMAKE_SKIP_BUILD_RPATH=OFF" # needed for tests
182 "-DCMAKE_INSTALL_RPATH=@loader_path/../lib" # needed for tools executables
183 ] ++ lib.optional (!stdenv.isx86_64) "-DARROW_USE_SIMD=OFF"
184 ++ lib.optional enableS3 "-DAWSSDK_CORE_HEADER_FILE=${aws-sdk-cpp}/include/aws/core/Aws.h";
185
186 doInstallCheck = true;
187 ARROW_TEST_DATA = lib.optionalString doInstallCheck "${arrow-testing}/data";
188 PARQUET_TEST_DATA = lib.optionalString doInstallCheck "${parquet-testing}/data";
189 GTEST_FILTER =
190 let
191 # Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11398
192 filteredTests = lib.optionals stdenv.hostPlatform.isAarch64 [
193 "TestFilterKernelWithNumeric/3.CompareArrayAndFilterRandomNumeric"
194 "TestFilterKernelWithNumeric/7.CompareArrayAndFilterRandomNumeric"
195 "TestCompareKernel.PrimitiveRandomTests"
196 ] ++ lib.optionals enableS3 [
197 "S3OptionsTest.FromUri"
198 "S3RegionResolutionTest.NonExistentBucket"
199 "S3RegionResolutionTest.PublicBucket"
200 "S3RegionResolutionTest.RestrictedBucket"
201 "TestMinioServer.Connect"
202 "TestS3FS.OpenOutputStreamBackgroundWrites"
203 "TestS3FS.OpenOutputStreamDestructorBackgroundWrites"
204 "TestS3FS.OpenOutputStreamDestructorSyncWrite"
205 "TestS3FS.OpenOutputStreamDestructorSyncWrites"
206 "TestS3FS.OpenOutputStreamMetadata"
207 "TestS3FS.OpenOutputStreamSyncWrites"
208 "TestS3FSGeneric.*"
209 ] ++ lib.optionals enableGcs [
210 "GcsFileSystem.FileSystemCompare"
211 "GcsIntegrationTest.*"
212 ];
213 in
214 lib.optionalString doInstallCheck "-${builtins.concatStringsSep ":" filteredTests}";
215 installCheckInputs = [ perl which ] ++ lib.optional enableS3 minio;
216 installCheckPhase =
217 let
218 excludedTests = lib.optionals stdenv.isDarwin [
219 # Some plasma tests need to be patched to use a shorter AF_UNIX socket
220 # path on Darwin. See https://github.com/NixOS/nix/pull/1085
221 "plasma-external-store-tests"
222 "plasma-client-tests"
223 ];
224 in
225 ''
226 runHook preInstallCheck
227
228 ctest -L unittest \
229 --exclude-regex '^(${builtins.concatStringsSep "|" excludedTests})$'
230
231 runHook postInstallCheck
232 '';
233
234 meta = with lib; {
235 description = "A cross-language development platform for in-memory data";
236 homepage = "https://arrow.apache.org/";
237 license = licenses.asl20;
238 platforms = platforms.unix;
239 maintainers = with maintainers; [ tobim veprbl cpcloud ];
240 };
241}