lol

hadoop: support native libs on 3.2

+31 -25
+31 -25
pkgs/applications/networking/cluster/hadoop/default.nix
··· 28 28 assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]; 29 29 30 30 let 31 - common = { pname, platformAttrs, untarDir ? "${pname}-${version}", jdk, openssl ? null, nativeLibs ? [ ], libPatchesGenerator ? (_: ""), tests }: 31 + common = { 32 + pname, platformAttrs, untarDir ? "${pname}-${version}", jdk 33 + , nativeLibs ? [ ], libPatchesGenerator ? (_: ""), tests 34 + }: 32 35 stdenv.mkDerivation (finalAttrs: { 33 - inherit pname jdk untarDir openssl; 36 + inherit pname jdk untarDir; 34 37 libPatches = libPatchesGenerator finalAttrs; 35 38 version = platformAttrs.${stdenv.system}.version or (throw "Unsupported system: ${stdenv.system}"); 36 39 src = fetchurl { 37 - url = "mirror://apache/hadoop/common/hadoop-${finalAttrs.version}/hadoop-${finalAttrs.version}" + optionalString stdenv.isAarch64 "-aarch64" + ".tar.gz"; 40 + url = "mirror://apache/hadoop/common/hadoop-${finalAttrs.version}/hadoop-${finalAttrs.version}" 41 + + optionalString stdenv.isAarch64 "-aarch64" + ".tar.gz"; 38 42 inherit (platformAttrs.${stdenv.system}) hash; 39 43 }; 40 44 doCheck = true; ··· 95 99 platforms = attrNames platformAttrs; 96 100 } (attrByPath [ stdenv.system "meta" ] {} platformAttrs); 97 101 }); 102 + nativeLibs = [ stdenv.cc.cc.lib protobuf zlib snappy libtirpc ]; 103 + libPatchesGenerator = finalAttrs: ('' 104 + ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${finalAttrs.untarDir}/lib/native/libsasl2.so.2 105 + ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${finalAttrs.untarDir}/lib/native/ 106 + ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/ 107 + ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/ 108 + ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/ 109 + '' + optionalString stdenv.isLinux '' 110 + # libjvm.so for Java >=11 111 + patchelf --add-rpath ${finalAttrs.jdk.home}/lib/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0 112 + # Java 8 has libjvm.so at a different path 113 + patchelf --add-rpath ${finalAttrs.jdk.home}/jre/lib/amd64/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0 114 + # NixOS/nixpkgs#193370 115 + # This workaround is needed to use protobuf 3.19 116 + # for hadoop 3.3 117 + patchelf --replace-needed libprotobuf.so.18 libprotobuf.so $out/lib/${finalAttrs.untarDir}/lib/native/libhdfspp.so 118 + # for hadoop 3.2 119 + patchelf --replace-needed libprotobuf.so.8 libprotobuf.so $out/lib/${finalAttrs.untarDir}/lib/native/libhdfspp.so 120 + patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \ 121 + $out/lib/${finalAttrs.untarDir}/lib/native/{libhdfspp.so.0.1.0,examples/{pipes-sort,wordcount-nopipe,wordcount-part,wordcount-simple}} 122 + ''); 98 123 in 99 124 { 100 125 # Different version of hadoop support different java runtime versions ··· 115 140 }; 116 141 untarDir = "${pname}-${platformAttrs.${stdenv.system}.version}"; 117 142 jdk = jdk11_headless; 118 - inherit openssl; 143 + inherit nativeLibs libPatchesGenerator; 119 144 # TODO: Package and add Intel Storage Acceleration Library 120 - nativeLibs = [ stdenv.cc.cc.lib protobuf zlib snappy libtirpc ]; 121 - libPatchesGenerator = finalAttrs: ('' 122 - ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${finalAttrs.untarDir}/lib/native/libsasl2.so.2 123 - ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${finalAttrs.untarDir}/lib/native/ 124 - ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/ 125 - ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/ 126 - ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/ 127 - '' + optionalString stdenv.isLinux '' 128 - # libjvm.so for Java >=11 129 - patchelf --add-rpath ${finalAttrs.jdk.home}/lib/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0 130 - # Java 8 has libjvm.so at a different path 131 - patchelf --add-rpath ${finalAttrs.jdk.home}/jre/lib/amd64/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0 132 - # NixOS/nixpkgs#193370 133 - # This workaround is needed to use protobuf 3.19 134 - patchelf --replace-needed libprotobuf.so.18 libprotobuf.so $out/lib/${finalAttrs.untarDir}/lib/native/libhdfspp.so 135 - patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \ 136 - $out/lib/${finalAttrs.untarDir}/lib/native/{libhdfspp.so.0.1.0,examples/{pipes-sort,wordcount-nopipe,wordcount-part,wordcount-simple}} 137 - ''); 138 145 tests = nixosTests.hadoop; 139 146 }; 140 - hadoop_3_2 = common rec { 147 + hadoop_3_2 = common { 141 148 pname = "hadoop"; 142 149 platformAttrs.x86_64-linux = { 143 150 version = "3.2.4"; ··· 145 152 srcHash = "sha256-F9nGD3mZZ1eJf3Ec3AJGE9YBcL/HiagskcdKQhCn/sw="; 146 153 }; 147 154 jdk = jdk8_headless; 148 - # not using native libs because of broken openssl_1_0_2 dependency 149 - # can be manually overridden 155 + inherit nativeLibs libPatchesGenerator; 150 156 tests = nixosTests.hadoop_3_2; 151 157 }; 152 158 hadoop2 = common rec {