lol

nixos/hadoop: add HBase submodule

authored by

illustris and committed by
pennae
ac403b83 cb2576c1

+297 -16
+1
nixos/modules/services/cluster/hadoop/conf.nix
··· 33 33 mkdir -p $out/ 34 34 cp ${siteXml "core-site.xml" (coreSite // coreSiteInternal)}/* $out/ 35 35 cp ${siteXml "hdfs-site.xml" (hdfsSiteDefault // hdfsSite // hdfsSiteInternal)}/* $out/ 36 + cp ${siteXml "hbase-site.xml" (hbaseSiteDefault // hbaseSite // hbaseSiteInternal)}/* $out/ 36 37 cp ${siteXml "mapred-site.xml" (mapredSiteDefault // mapredSite)}/* $out/ 37 38 cp ${siteXml "yarn-site.xml" (yarnSiteDefault // yarnSite // yarnSiteInternal)}/* $out/ 38 39 cp ${siteXml "httpfs-site.xml" httpfsSite}/* $out/
+1 -1
nixos/modules/services/cluster/hadoop/default.nix
··· 5 5 in 6 6 with lib; 7 7 { 8 - imports = [ ./yarn.nix ./hdfs.nix ]; 8 + imports = [ ./yarn.nix ./hdfs.nix ./hbase.nix ]; 9 9 10 10 options.services.hadoop = { 11 11 coreSite = mkOption {
+200
nixos/modules/services/cluster/hadoop/hbase.nix
··· 1 + { config, lib, pkgs, ...}: 2 + 3 + with lib; 4 + let 5 + cfg = config.services.hadoop; 6 + hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/"; 7 + mkIfNotNull = x: mkIf (x != null) x; 8 + in 9 + { 10 + options.services.hadoop = { 11 + 12 + gatewayRole.enableHbaseCli = mkOption { 13 + description = "Whether to enable HBase CLI tools"; 14 + default = false; 15 + type = types.bool; 16 + }; 17 + 18 + hbaseSiteDefault = mkOption { 19 + default = { 20 + "hbase.regionserver.ipc.address" = "0.0.0.0"; 21 + "hbase.master.ipc.address" = "0.0.0.0"; 22 + "hbase.master.info.bindAddress" = "0.0.0.0"; 23 + "hbase.regionserver.info.bindAddress" = "0.0.0.0"; 24 + 25 + "hbase.cluster.distributed" = "true"; 26 + }; 27 + type = types.attrsOf types.anything; 28 + description = '' 29 + Default options for hbase-site.xml 30 + ''; 31 + }; 32 + hbaseSite = mkOption { 33 + default = {}; 34 + type = with types; attrsOf anything; 35 + example = literalExpression '' 36 + ''; 37 + description = '' 38 + Additional options and overrides for hbase-site.xml 39 + <link xlink:href="https://github.com/apache/hbase/blob/rel/2.4.11/hbase-common/src/main/resources/hbase-default.xml"/> 40 + ''; 41 + }; 42 + hbaseSiteInternal = mkOption { 43 + default = {}; 44 + type = with types; attrsOf anything; 45 + internal = true; 46 + description = '' 47 + Internal option to add configs to hbase-site.xml based on module options 48 + ''; 49 + }; 50 + 51 + hbase = { 52 + 53 + package = mkOption { 54 + type = types.package; 55 + default = pkgs.hbase; 56 + defaultText = literalExpression "pkgs.hbase"; 57 + description = "HBase package"; 58 + }; 59 + 60 + rootdir = mkOption { 61 + description = '' 62 + This option will set "hbase.rootdir" in hbase-site.xml and determine 63 + the directory shared by region servers and into which HBase persists. 64 + The URL should be 'fully-qualified' to include the filesystem scheme. 65 + If a core-site.xml is provided, the FS scheme defaults to the value 66 + of "fs.defaultFS". 67 + 68 + Filesystems other than HDFS (like S3, QFS, Swift) are also supported. 69 + ''; 70 + type = types.str; 71 + example = "hdfs://nameservice1/hbase"; 72 + default = "/hbase"; 73 + }; 74 + zookeeperQuorum = mkOption { 75 + description = '' 76 + This option will set "hbase.zookeeper.quorum" in hbase-site.xml. 77 + Comma separated list of servers in the ZooKeeper ensemble. 78 + ''; 79 + type = with types; nullOr commas; 80 + example = "zk1.internal,zk2.internal,zk3.internal"; 81 + default = null; 82 + }; 83 + master = { 84 + enable = mkEnableOption "HBase Master"; 85 + initHDFS = mkEnableOption "initialization of the hbase directory on HDFS"; 86 + 87 + openFirewall = mkOption { 88 + type = types.bool; 89 + default = false; 90 + description = '' 91 + Open firewall ports for HBase master. 92 + ''; 93 + }; 94 + }; 95 + regionServer = { 96 + enable = mkEnableOption "HBase RegionServer"; 97 + 98 + overrideHosts = mkOption { 99 + type = types.bool; 100 + default = true; 101 + description = '' 102 + Remove /etc/hosts entries for "127.0.0.2" and "::1" defined in nixos/modules/config/networking.nix 103 + Regionservers must be able to resolve their hostnames to their IP addresses, through PTR records 104 + or /etc/hosts entries. 105 + 106 + ''; 107 + }; 108 + 109 + openFirewall = mkOption { 110 + type = types.bool; 111 + default = false; 112 + description = '' 113 + Open firewall ports for HBase master. 114 + ''; 115 + }; 116 + }; 117 + }; 118 + }; 119 + 120 + config = mkMerge [ 121 + (mkIf cfg.hbase.master.enable { 122 + services.hadoop.gatewayRole = { 123 + enable = true; 124 + enableHbaseCli = mkDefault true; 125 + }; 126 + 127 + systemd.services.hbase-master = { 128 + description = "HBase master"; 129 + wantedBy = [ "multi-user.target" ]; 130 + 131 + preStart = mkIf cfg.hbase.master.initHDFS '' 132 + HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfsadmin -safemode wait 133 + HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfs -mkdir -p ${cfg.hbase.rootdir} 134 + HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfs -chown hbase ${cfg.hbase.rootdir} 135 + ''; 136 + 137 + serviceConfig = { 138 + User = "hbase"; 139 + SyslogIdentifier = "hbase-master"; 140 + ExecStart = "${cfg.hbase.package}/bin/hbase --config ${hadoopConf} " + 141 + "master start"; 142 + Restart = "always"; 143 + }; 144 + }; 145 + 146 + services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir; 147 + 148 + networking.firewall.allowedTCPPorts = (mkIf cfg.hbase.master.openFirewall [ 149 + 16000 16010 150 + ]); 151 + 152 + }) 153 + 154 + (mkIf cfg.hbase.regionServer.enable { 155 + services.hadoop.gatewayRole = { 156 + enable = true; 157 + enableHbaseCli = mkDefault true; 158 + }; 159 + 160 + systemd.services.hbase-regionserver = { 161 + description = "HBase RegionServer"; 162 + wantedBy = [ "multi-user.target" ]; 163 + serviceConfig = { 164 + User = "hbase"; 165 + SyslogIdentifier = "hbase-regionserver"; 166 + ExecStart = "${cfg.hbase.package}/bin/hbase --config /etc/hadoop-conf/ " + 167 + "regionserver start"; 168 + Restart = "always"; 169 + }; 170 + }; 171 + 172 + services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir; 173 + 174 + networking = { 175 + firewall.allowedTCPPorts = (mkIf cfg.hbase.regionServer.openFirewall [ 176 + 16020 16030 177 + ]); 178 + hosts = mkIf cfg.hbase.regionServer.overrideHosts { 179 + "127.0.0.2" = mkForce [ ]; 180 + "::1" = mkForce [ ]; 181 + }; 182 + }; 183 + }) 184 + 185 + (mkIf cfg.gatewayRole.enable { 186 + 187 + environment.systemPackages = mkIf cfg.gatewayRole.enableHbaseCli [ cfg.hbase.package ]; 188 + 189 + services.hadoop.hbaseSiteInternal = with cfg.hbase; { 190 + "hbase.zookeeper.quorum" = mkIfNotNull zookeeperQuorum; 191 + }; 192 + 193 + users.users.hbase = { 194 + description = "Hadoop HBase user"; 195 + group = "hadoop"; 196 + isSystemUser = true; 197 + }; 198 + }) 199 + ]; 200 + }
+8 -13
nixos/modules/services/databases/hbase.nix
··· 3 3 with lib; 4 4 5 5 let 6 - cfg = config.services.hbase; 7 - opt = options.services.hbase; 6 + cfg = config.services.hbase-standalone; 7 + opt = options.services.hbase-standalone; 8 8 9 9 buildProperty = configAttr: 10 10 (builtins.concatStringsSep "\n" ··· 35 35 ###### interface 36 36 37 37 options = { 38 - 39 - services.hbase = { 38 + services.hbase-standalone = { 40 39 41 - enable = mkOption { 42 - type = types.bool; 43 - default = false; 44 - description = lib.mdDoc '' 45 - Whether to run HBase. 46 - ''; 47 - }; 40 + enable = mkEnableOption '' 41 + HBase master in standalone mode with embedded regionserver and zookeper. 42 + Do not use this configuration for production nor for evaluating HBase performance. 43 + ''; 48 44 49 45 package = mkOption { 50 46 type = types.package; ··· 108 104 }; 109 105 110 106 }; 111 - 112 107 }; 113 108 114 109 ###### implementation 115 110 116 - config = mkIf config.services.hbase.enable { 111 + config = mkIf cfg.enable { 117 112 118 113 systemd.tmpfiles.rules = [ 119 114 "d '${cfg.dataDir}' - ${cfg.user} ${cfg.group} - -"
+1
nixos/tests/hadoop/default.nix
··· 4 4 all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { inherit package; }; 5 5 hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { inherit package; }; 6 6 yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { inherit package; }; 7 + hbase = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hbase.nix { inherit package; }; 7 8 }
+84
nixos/tests/hadoop/hbase.nix
··· 1 + # Test a minimal hbase cluster 2 + { pkgs, ... }: 3 + import ../make-test-python.nix ({ hadoop ? pkgs.hadoop, hbase ? pkgs.hbase, ... }: 4 + with pkgs.lib; 5 + { 6 + name = "hadoop-hbase"; 7 + 8 + nodes = let 9 + coreSite = { 10 + "fs.defaultFS" = "hdfs://namenode:8020"; 11 + }; 12 + defOpts = { 13 + enable = true; 14 + openFirewall = true; 15 + }; 16 + zookeeperQuorum = "zookeeper"; 17 + in { 18 + zookeeper = { ... }: { 19 + services.zookeeper.enable = true; 20 + networking.firewall.allowedTCPPorts = [ 2181 ]; 21 + }; 22 + namenode = { ... }: { 23 + services.hadoop = { 24 + hdfs = { 25 + namenode = defOpts // { formatOnInit = true; }; 26 + }; 27 + inherit coreSite; 28 + }; 29 + }; 30 + datanode = { ... }: { 31 + virtualisation.diskSize = 8192; 32 + services.hadoop = { 33 + hdfs.datanode = defOpts; 34 + inherit coreSite; 35 + }; 36 + }; 37 + 38 + master = { ... }:{ 39 + services.hadoop = { 40 + inherit coreSite; 41 + hbase = { 42 + inherit zookeeperQuorum; 43 + master = defOpts // { initHDFS = true; }; 44 + }; 45 + }; 46 + }; 47 + regionserver = { ... }:{ 48 + services.hadoop = { 49 + inherit coreSite; 50 + hbase = { 51 + inherit zookeeperQuorum; 52 + regionServer = defOpts; 53 + }; 54 + }; 55 + }; 56 + }; 57 + 58 + testScript = '' 59 + start_all() 60 + 61 + # wait for HDFS cluster 62 + namenode.wait_for_unit("hdfs-namenode") 63 + namenode.wait_for_unit("network.target") 64 + namenode.wait_for_open_port(8020) 65 + namenode.wait_for_open_port(9870) 66 + datanode.wait_for_unit("hdfs-datanode") 67 + datanode.wait_for_unit("network.target") 68 + datanode.wait_for_open_port(9864) 69 + datanode.wait_for_open_port(9866) 70 + datanode.wait_for_open_port(9867) 71 + 72 + # wait for ZK 73 + zookeeper.wait_for_unit("zookeeper") 74 + zookeeper.wait_for_open_port(2181) 75 + 76 + # wait for HBase to start up 77 + master.wait_for_unit("hbase-master") 78 + regionserver.wait_for_unit("hbase-regionserver") 79 + 80 + assert "1 active master, 0 backup masters, 1 servers" in master.succeed("echo status | HADOOP_USER_NAME=hbase hbase shell -n") 81 + regionserver.wait_until_succeeds("echo \"create 't1','f1'\" | HADOOP_USER_NAME=hbase hbase shell -n") 82 + assert "NAME => 'f1'" in regionserver.succeed("echo \"describe 't1'\" | HADOOP_USER_NAME=hbase hbase shell -n") 83 + ''; 84 + })
+2 -2
nixos/tests/hbase.nix
··· 1 1 import ./make-test-python.nix ({ pkgs, lib, package ? pkgs.hbase, ... }: 2 2 { 3 - name = "hbase"; 3 + name = "hbase-standalone"; 4 4 5 5 meta = with lib.maintainers; { 6 6 maintainers = [ illustris ]; ··· 8 8 9 9 nodes = { 10 10 hbase = { pkgs, ... }: { 11 - services.hbase = { 11 + services.hbase-standalone = { 12 12 enable = true; 13 13 inherit package; 14 14 # Needed for standalone mode in hbase 2+