tangled
alpha
login
or
join now
pyrox.dev
/
nixpkgs
0
fork
atom
lol
0
fork
atom
overview
issues
pulls
pipelines
hadoop: support native libs on 3.2
illustris
2 years ago
9be640a2
5dfa639d
+31
-25
1 changed file
expand all
collapse all
unified
split
pkgs
applications
networking
cluster
hadoop
default.nix
+31
-25
pkgs/applications/networking/cluster/hadoop/default.nix
reviewed
···
28
28
assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];
29
29
30
30
let
31
31
-
common = { pname, platformAttrs, untarDir ? "${pname}-${version}", jdk, openssl ? null, nativeLibs ? [ ], libPatchesGenerator ? (_: ""), tests }:
31
31
+
common = {
32
32
+
pname, platformAttrs, untarDir ? "${pname}-${version}", jdk
33
33
+
, nativeLibs ? [ ], libPatchesGenerator ? (_: ""), tests
34
34
+
}:
32
35
stdenv.mkDerivation (finalAttrs: {
33
33
-
inherit pname jdk untarDir openssl;
36
36
+
inherit pname jdk untarDir;
34
37
libPatches = libPatchesGenerator finalAttrs;
35
38
version = platformAttrs.${stdenv.system}.version or (throw "Unsupported system: ${stdenv.system}");
36
39
src = fetchurl {
37
37
-
url = "mirror://apache/hadoop/common/hadoop-${finalAttrs.version}/hadoop-${finalAttrs.version}" + optionalString stdenv.isAarch64 "-aarch64" + ".tar.gz";
40
40
+
url = "mirror://apache/hadoop/common/hadoop-${finalAttrs.version}/hadoop-${finalAttrs.version}"
41
41
+
+ optionalString stdenv.isAarch64 "-aarch64" + ".tar.gz";
38
42
inherit (platformAttrs.${stdenv.system}) hash;
39
43
};
40
44
doCheck = true;
···
95
99
platforms = attrNames platformAttrs;
96
100
} (attrByPath [ stdenv.system "meta" ] {} platformAttrs);
97
101
});
102
102
+
nativeLibs = [ stdenv.cc.cc.lib protobuf zlib snappy libtirpc ];
103
103
+
libPatchesGenerator = finalAttrs: (''
104
104
+
ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${finalAttrs.untarDir}/lib/native/libsasl2.so.2
105
105
+
ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${finalAttrs.untarDir}/lib/native/
106
106
+
ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
107
107
+
ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
108
108
+
ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
109
109
+
'' + optionalString stdenv.isLinux ''
110
110
+
# libjvm.so for Java >=11
111
111
+
patchelf --add-rpath ${finalAttrs.jdk.home}/lib/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0
112
112
+
# Java 8 has libjvm.so at a different path
113
113
+
patchelf --add-rpath ${finalAttrs.jdk.home}/jre/lib/amd64/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0
114
114
+
# NixOS/nixpkgs#193370
115
115
+
# This workaround is needed to use protobuf 3.19
116
116
+
# for hadoop 3.3
117
117
+
patchelf --replace-needed libprotobuf.so.18 libprotobuf.so $out/lib/${finalAttrs.untarDir}/lib/native/libhdfspp.so
118
118
+
# for hadoop 3.2
119
119
+
patchelf --replace-needed libprotobuf.so.8 libprotobuf.so $out/lib/${finalAttrs.untarDir}/lib/native/libhdfspp.so
120
120
+
patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \
121
121
+
$out/lib/${finalAttrs.untarDir}/lib/native/{libhdfspp.so.0.1.0,examples/{pipes-sort,wordcount-nopipe,wordcount-part,wordcount-simple}}
122
122
+
'');
98
123
in
99
124
{
100
125
# Different version of hadoop support different java runtime versions
···
115
140
};
116
141
untarDir = "${pname}-${platformAttrs.${stdenv.system}.version}";
117
142
jdk = jdk11_headless;
118
118
-
inherit openssl;
143
143
+
inherit nativeLibs libPatchesGenerator;
119
144
# TODO: Package and add Intel Storage Acceleration Library
120
120
-
nativeLibs = [ stdenv.cc.cc.lib protobuf zlib snappy libtirpc ];
121
121
-
libPatchesGenerator = finalAttrs: (''
122
122
-
ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${finalAttrs.untarDir}/lib/native/libsasl2.so.2
123
123
-
ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${finalAttrs.untarDir}/lib/native/
124
124
-
ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
125
125
-
ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
126
126
-
ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
127
127
-
'' + optionalString stdenv.isLinux ''
128
128
-
# libjvm.so for Java >=11
129
129
-
patchelf --add-rpath ${finalAttrs.jdk.home}/lib/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0
130
130
-
# Java 8 has libjvm.so at a different path
131
131
-
patchelf --add-rpath ${finalAttrs.jdk.home}/jre/lib/amd64/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0
132
132
-
# NixOS/nixpkgs#193370
133
133
-
# This workaround is needed to use protobuf 3.19
134
134
-
patchelf --replace-needed libprotobuf.so.18 libprotobuf.so $out/lib/${finalAttrs.untarDir}/lib/native/libhdfspp.so
135
135
-
patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \
136
136
-
$out/lib/${finalAttrs.untarDir}/lib/native/{libhdfspp.so.0.1.0,examples/{pipes-sort,wordcount-nopipe,wordcount-part,wordcount-simple}}
137
137
-
'');
138
145
tests = nixosTests.hadoop;
139
146
};
140
140
-
hadoop_3_2 = common rec {
147
147
+
hadoop_3_2 = common {
141
148
pname = "hadoop";
142
149
platformAttrs.x86_64-linux = {
143
150
version = "3.2.4";
···
145
152
srcHash = "sha256-F9nGD3mZZ1eJf3Ec3AJGE9YBcL/HiagskcdKQhCn/sw=";
146
153
};
147
154
jdk = jdk8_headless;
148
148
-
# not using native libs because of broken openssl_1_0_2 dependency
149
149
-
# can be manually overridden
155
155
+
inherit nativeLibs libPatchesGenerator;
150
156
tests = nixosTests.hadoop_3_2;
151
157
};
152
158
hadoop2 = common rec {