Merge pull request #114456 from thefloweringash/elk-update

elk: 7.5.1 -> 7.11.1

nixos/tests/elk.nix

···
+# To run the test on the unfree ELK use the folllowing command:
+# cd path/to/nixpkgs
+# NIXPKGS_ALLOW_UNFREE=1 nix-build -A nixosTests.elk.unfree.ELK-6
+
 { system ? builtins.currentSystem,
   config ? {},
   pkgs ? import ../.. { inherit system config; },
-  enableUnfree ? false
-  # To run the test on the unfree ELK use the folllowing command:
-  # NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true
 }:
 
 let
+  inherit (pkgs) lib;
+
   esUrl = "http://localhost:9200";
 
   mkElkTest = name : elk :
···
               '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^'
           )
     '';
-  }) {};
-in pkgs.lib.mapAttrs mkElkTest {
-  ELK-6 =
-    if enableUnfree
-    then {
+  }) { inherit pkgs system; };
+in {
+  ELK-6 = mkElkTest {
+    name = "elk-6-oss";
+    elasticsearch = pkgs.elasticsearch6-oss;
+    logstash      = pkgs.logstash6-oss;
+    kibana        = pkgs.kibana6-oss;
+    journalbeat   = pkgs.journalbeat6;
+    metricbeat    = pkgs.metricbeat6;
+  };
+  # We currently only package upstream binaries.
+  # Feel free to package an SSPL licensed source-based package!
+  # ELK-7 = mkElkTest {
+  #   name = "elk-7";
+  #   elasticsearch = pkgs.elasticsearch7-oss;
+  #   logstash      = pkgs.logstash7-oss;
+  #   kibana        = pkgs.kibana7-oss;
+  #   journalbeat   = pkgs.journalbeat7;
+  #   metricbeat    = pkgs.metricbeat7;
+  # };
+  unfree = lib.dontRecurseIntoAttrs {
+    ELK-6 = mkElkTest "elk-6" {
       elasticsearch = pkgs.elasticsearch6;
       logstash      = pkgs.logstash6;
       kibana        = pkgs.kibana6;
       journalbeat   = pkgs.journalbeat6;
       metricbeat    = pkgs.metricbeat6;
-    }
-    else {
-      elasticsearch = pkgs.elasticsearch6-oss;
-      logstash      = pkgs.logstash6-oss;
-      kibana        = pkgs.kibana6-oss;
-      journalbeat   = pkgs.journalbeat6;
-      metricbeat    = pkgs.metricbeat6;
     };
-  ELK-7 =
-    if enableUnfree
-    then {
+    ELK-7 = mkElkTest "elk-7" {
       elasticsearch = pkgs.elasticsearch7;
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
       journalbeat   = pkgs.journalbeat7;
       metricbeat    = pkgs.metricbeat7;
-    }
-    else {
-      elasticsearch = pkgs.elasticsearch7-oss;
-      logstash      = pkgs.logstash7-oss;
-      kibana        = pkgs.kibana7-oss;
-      journalbeat   = pkgs.journalbeat7;
-      metricbeat    = pkgs.metricbeat7;
     };
+  };
 }

pkgs/development/tools/misc/kibana/7.x.nix

···
 , stdenv
 , makeWrapper
 , fetchurl
-, nodejs-10_x
+, nodejs-14_x
 , coreutils
 , which
 }:
 
 with lib;
 let
-  nodejs = nodejs-10_x;
+  nodejs = nodejs-14_x;
   inherit (builtins) elemAt;
   info = splitString "-" stdenv.hostPlatform.system;
   arch = elemAt info 0;
   plat = elemAt info 1;
   shas =
-    if enableUnfree
-    then {
-      x86_64-linux = "sha256-lTPBppKm51zgKSQtSdO0PgZ/aomvaStwqwYYGNPY4Bo=";
-      x86_64-darwin = "sha256-d7xHmoASiywDlZCJX/CfUX1VIi4iOcDrqvK0su54MJc=";
-    }
-    else {
-      x86_64-linux = "sha256-+pkKpiXBpLHs72KKNtMJbqipw6eu5XC1xu/iLFCHGRQ=";
-      x86_64-darwin = "sha256-CyJ5iRXaPgXO2lyy+E24OcGtb9V3e1gMZRIu25bVyzk=";
+    {
+      x86_64-linux  = "19p9s4sir982bb1zcldrbphhwfs9i11p0q28vgc421iqg10kjlf1";
+      x86_64-darwin = "0qq557ngwwakifidyrccga4cadj9k9pzhjwy4msmbcgf5pb86qyc";
+      aarch64-linux = "183cp1h8d3n7xfcpcys4hf36palczxa409afyp62kzyzckngy0j8";
     };
 
-in
-stdenv.mkDerivation rec {
-  pname = "kibana${optionalString (!enableUnfree) "-oss"}";
+in stdenv.mkDerivation rec {
+  pname = "kibana";
   version = elk7Version;
 
   src = fetchurl {
···
   meta = {
     description = "Visualize logs and time-stamped data";
     homepage = "http://www.elasticsearch.org/overview/kibana";
-    license = if enableUnfree then licenses.elastic else licenses.asl20;
+    license = licenses.elastic;
     maintainers = with maintainers; [ offline basvandijk ];
     platforms = with platforms; unix;
   };

pkgs/misc/logging/beats/7.x.nix

···
     owner = "elastic";
     repo = "beats";
     rev = "v${version}";
-    sha256 = "sha256-zr0a0LBR4G9okS2pUixDYtYZ0yCp4G6j08jx/zlIKOA=";
+    sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8";
   };
 
-  vendorSha256 = "sha256-xmw432vY1T2EixkDcXdGrnMdc8fYOI4R2lEjbkav3JQ=";
+  vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c";
 
   subPackages = [ package ];
 

pkgs/servers/search/elasticsearch/7.x.nix

···
 { elk7Version
-, enableUnfree ? true
 , lib
 , stdenv
 , fetchurl
···
   arch = elemAt info 0;
   plat = elemAt info 1;
   shas =
-    if enableUnfree
-    then {
-      x86_64-linux = "sha256-O3rjtvXyJI+kRBqiz2U2OMkCIQj4E+AIHaE8N4o14R4=";
-      x86_64-darwin = "sha256-AwuY2yMxf+v7U5/KD3Cf+Hv6ijjySEyj6pzF3RCsg24=";
-    }
-    else {
-      x86_64-linux = "sha256-cJrdkFIFgAI6wfQh34Z8yFuLrOCOKzgOsWZhU3S/3NQ=";
-      x86_64-darwin = "sha256-OhMVOdXei9D9cH+O5tBhdKvZ05TsImjMqUUsucRyWMo=";
+    {
+      x86_64-linux  = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10";
+      x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3";
+      aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy";
     };
 in
-stdenv.mkDerivation (rec {
+stdenv.mkDerivation rec {
+  pname = "elasticsearch";
   version = elk7Version;
-  pname = "elasticsearch${optionalString (!enableUnfree) "-oss"}";
 
   src = fetchurl {
     url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}-${plat}-${arch}.tar.gz";
···
       "ES_CLASSPATH=\"\$ES_CLASSPATH:$out/\$additional_classpath_directory/*\""
   '';
 
-  nativeBuildInputs = [ makeWrapper ];
-  buildInputs = [ jre_headless util-linux ]
-    ++ optional enableUnfree zlib;
+  nativeBuildInputs = [ autoPatchelfHook makeWrapper ];
+
+  buildInputs = [ jre_headless util-linux zlib ];
+
+  runtimeDependencies = [ zlib ];
 
   installPhase = ''
     mkdir -p $out
···
     wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
   '';
 
-  passthru = { inherit enableUnfree; };
+  passthru = { enableUnfree = true; };
 
   meta = {
     description = "Open Source, Distributed, RESTful Search Engine";
-    license = if enableUnfree then licenses.elastic else licenses.asl20;
+    license = licenses.elastic;
     platforms = platforms.unix;
     maintainers = with maintainers; [ apeschar basvandijk ];
   };
-} // optionalAttrs enableUnfree {
-  dontPatchELF = true;
-  nativeBuildInputs = [ makeWrapper autoPatchelfHook ];
-  runtimeDependencies = [ zlib ];
-  postFixup = ''
-    for exe in $(find $out/modules/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do
-      echo "patching $exe..."
-      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe"
-    done
-  '';
-})
+}

pkgs/servers/search/elasticsearch/plugins.nix

···
   esVersion = elasticsearch.version;
 
   esPlugin =
-    a@{ pluginName
-    , installPhase ? ''
+    a@{
+      pluginName,
+      installPhase ? ''
         mkdir -p $out/config
         mkdir -p $out/plugins
-        ln -s ${elasticsearch}/lib $out/lib
+        ln -s ${elasticsearch}/lib ${elasticsearch}/modules $out
         ES_HOME=$out ${elasticsearch}/bin/elasticsearch-plugin install --batch -v file://$src
-        rm $out/lib
+        rm $out/lib $out/modules
       ''
     , ...
     }:
···
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.10.2" then "sha256-HXNJy8WPExPeh5afjdLEFg+0WX0LYI/kvvaLGVUke5E="
+        if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy"
         else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
···
     src = fetchurl {
       url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip";
       sha256 =
-        if version == "7.10.2" then "sha256-mW4YNZ20qatyfHCDAmod/gVmkPYh15NrsYPgiBy1/T8="
+        if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3"
         else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
···
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.10.2" then "sha256-PjA/pwoulkD2d6sHKqzcYxQpb1aS68/l047z5JTcV3Y="
+        if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855"
         else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
···
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.10.2" then "sha256-yvxSkVyZDWeu7rcxxq1+IVsljZQKgWEURiXY9qycK1s="
+        if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6"
         else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
···
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.10.2" then "sha256-yOMiYJ2c/mcLDcTA99YrpQBiEBAa/mLtTqJlqTJ5tBc="
+        if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs"
         else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
···
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.10.2" then "sha256-fN2RQsY9OACE71pIw87XVJo4c3sUu/6gf/6wUt7ZNIE="
+        if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs"
         else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
···
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.10.2" then "sha256-JdWt5LzSbs0MIEuLJIE1ceTnNeTYI5Jt2N0Xj7OBO6g="
+        if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7"
         else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
···
     };
   };
 
-  search-guard =
-    let
-      majorVersion = lib.head (builtins.splitVersion esVersion);
-    in
-    esPlugin rec {
-      pluginName = "search-guard";
-      version =
-        # https://docs.search-guard.com/latest/search-guard-versions
-        if esVersion == "7.10.2" then "7.10.1-49.3.0"
-        else if esVersion == "6.8.3" then "${esVersion}-25.5"
-        else throw "unsupported version ${esVersion} for plugin ${pluginName}";
-      src = fetchurl {
-        url =
-          if version == "7.10.1-49.3.0" then "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip"
-          else "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip";
-        sha256 =
-          if version == "7.10.1-49.3.0" then "sha256-vKH2+c+7WlncgljrvYH9lAqQTKzg9l0ABZ23Q/xdoK4="
-          else if version == "6.8.3-25.5" then "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb"
-          else throw "unsupported version ${version} for plugin ${pluginName}";
-      };
-      meta = with lib; {
-        homepage = "https://search-guard.com";
-        description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. ";
-        license = licenses.asl20;
-      };
+  search-guard = let
+    majorVersion = lib.head (builtins.splitVersion esVersion);
+  in esPlugin rec {
+    pluginName = "search-guard";
+    version =
+      # https://docs.search-guard.com/latest/search-guard-versions
+      if esVersion == "7.11.1" then "${esVersion}-50.0.0"
+      else if esVersion == "6.8.3" then "${esVersion}-25.5"
+      else throw "unsupported version ${esVersion} for plugin ${pluginName}";
+    src =
+      if esVersion == "7.11.1" then
+        fetchurl {
+          url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip";
+          sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm";
+        }
+      else if esVersion == "6.8.3" then
+        fetchurl {
+          url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip";
+          sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb";
+        }
+      else throw "unsupported version ${version} for plugin ${pluginName}";
+    meta = with lib; {
+      homepage = "https://search-guard.com";
+      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. ";
+      license = licenses.asl20;
     };
+  };
 }

pkgs/tools/misc/logstash/7.x.nix

···
   shas =
     if enableUnfree
     then {
-      x86_64-linux = "sha256-5qv4fbFpLf6aduD7wyxXQ6FsCeUqrszRisNBx44vbMY=";
-      x86_64-darwin = "sha256-7H+Xpo8qF1ZZMkR5n92PVplEN4JsBEYar91zHQhE+Lo=";
+      x86_64-linux  = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
+      x86_64-darwin = "0dqm66c89w1nvmbwqzphlqmf7avrycgv1nwd5b0k1z168fj0c3zm";
+      aarch64-linux = "11hjhyb48mjagmvqyxb780n57kr619h6p4adl2vs1zm97g9gslx8";
     }
     else {
-      x86_64-linux = "sha256-jiV2yGPwPgZ5plo3ftImVDLSOsk/XBzFkeeALSObLhU=";
-      x86_64-darwin = "sha256-UYG+GGr23eAc2GgNX/mXaGU0WKMjiQMPpD1wUvAVz0A=";
+      x86_64-linux  = "14b1649avjcalcsi0ffkgznq6d93qdk6m3j0i73mwfqka5d3dvy3";
+      x86_64-darwin = "0ypgdfklr5rxvsnc3czh231pa1z2h70366j1c6q5g64b3xnxpphs";
+      aarch64-linux = "01ainayr8fwwfix7dmxfhhmb23ji65dn4lbjwnj2w0pl0ym9h9w2";
     };
   this = stdenv.mkDerivation rec {
     version = elk7Version;
     pname = "logstash${optionalString (!enableUnfree) "-oss"}";
+
 
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}-${plat}-${arch}.tar.gz";

pkgs/top-level/aliases.nix

···
   ec2_ami_tools = ec2-ami-tools; # added 2021-10-08
   ec2_api_tools = ec2-api-tools; # added 2021-10-08
   elasticmq = throw "elasticmq has been removed in favour of elasticmq-server-bin"; # added 2021-01-17
+  elasticsearch7-oss = throw "elasticsearch7-oss has been removed, as the distribution is no longer provided by upstream. https://github.com/NixOS/nixpkgs/pull/114456"; # added 2021-06-09
   emacsPackagesGen = emacsPackagesFor; # added 2018-08-18
   emacsPackagesNgGen = emacsPackagesFor; # added 2018-08-18
   emacsPackagesNgFor = emacsPackagesFor; # added 2019-08-07
···
   json_glib = json-glib; # added 2018-02-25
   kdecoration-viewer = throw "kdecoration-viewer has been removed from nixpkgs, as there is no upstream activity"; # 2020-06-16
   k9copy = throw "k9copy has been removed from nixpkgs, as there is no upstream activity"; # 2020-11-06
+  kibana7-oss = throw "kibana7-oss has been removed, as the distribution is no longer provided by upstream. https://github.com/NixOS/nixpkgs/pull/114456"; # added 2021-06-09
   kodiGBM = kodi-gbm;
   kodiPlain = kodi;
   kodiPlainWayland = kodi-wayland;

pkgs/top-level/all-packages.nix

···
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   # When updating make sure to update all plugins or they will break!
   elk6Version = "6.8.3";
-  elk7Version = "7.10.2";
+  elk7Version = "7.11.1";
 
   elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix {
     util-linux = util-linuxMinimal;
···
     util-linux = util-linuxMinimal;
     jre_headless = jdk11_headless; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731
   };
-  elasticsearch7-oss = callPackage ../servers/search/elasticsearch/7.x.nix {
-    enableUnfree = false;
-    util-linux = util-linuxMinimal;
-    jre_headless = jdk11_headless; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731
-  };
   elasticsearch = elasticsearch6;
   elasticsearch-oss = elasticsearch6-oss;
 
···
     elasticsearch = elasticsearch6-oss;
   };
   elasticsearch7Plugins = elasticsearchPlugins.override {
-    elasticsearch = elasticsearch7-oss;
+    elasticsearch = elasticsearch7;
   };
 
   elasticsearch-curator = callPackage ../tools/admin/elasticsearch-curator {
···
     enableUnfree = false;
   };
   kibana7 = callPackage ../development/tools/misc/kibana/7.x.nix { };
-  kibana7-oss = callPackage ../development/tools/misc/kibana/7.x.nix {
-    enableUnfree = false;
-  };
   kibana = kibana6;
   kibana-oss = kibana6-oss;