···9394- `git-bug` has been updated to at least version 0.8.0, which includes backwards incompatible changes. The `git-bug-migration` package can be used to upgrade existing repositories.
950096- `keepassx` and `keepassx2` have been removed, due to upstream [stopping development](https://www.keepassx.org/index.html%3Fp=636.html). Consider [KeePassXC](https://keepassxc.org) as a maintained alternative.
9798- The `services.kubo.settings` option is now no longer stateful. If you changed any of the options in `services.kubo.settings` in the past and then removed them from your NixOS configuration again, those changes are still in your Kubo configuration file but will now be reset to the default. If you're unsure, you may want to make a backup of your configuration file (probably /var/lib/ipfs/config) and compare after the update.
···9394- `git-bug` has been updated to at least version 0.8.0, which includes backwards incompatible changes. The `git-bug-migration` package can be used to upgrade existing repositories.
9596+- `nushell` has been updated to at least version 0.77.0, which includes potential breaking changes in aliases. The old aliases are now available as `old-alias` but it is recommended you migrate to the new format. See [Reworked aliases](https://www.nushell.sh/blog/2023-03-14-nushell_0_77.html#reworked-aliases-breaking-changes-kubouch).
97+98- `keepassx` and `keepassx2` have been removed, due to upstream [stopping development](https://www.keepassx.org/index.html%3Fp=636.html). Consider [KeePassXC](https://keepassxc.org) as a maintained alternative.
99100- The `services.kubo.settings` option is now no longer stateful. If you changed any of the options in `services.kubo.settings` in the past and then removed them from your NixOS configuration again, those changes are still in your Kubo configuration file but will now be reset to the default. If you're unsure, you may want to make a backup of your configuration file (probably /var/lib/ipfs/config) and compare after the update.
+125-103
nixos/modules/services/cluster/hadoop/hbase.nix
···5 cfg = config.services.hadoop;
6 hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
7 mkIfNotNull = x: mkIf (x != null) x;
0000000000000000000000000000000000000000000000000000000000000000000000000000000000008in
9{
10 options.services.hadoop = {
1112- gatewayRole.enableHbaseCli = mkEnableOption (lib.mdDoc "HBase CLI tools");
1314 hbaseSiteDefault = mkOption {
15 default = {
···21 "hbase.cluster.distributed" = "true";
22 };
23 type = types.attrsOf types.anything;
24- description = lib.mdDoc ''
25 Default options for hbase-site.xml
26 '';
27 };
···29 default = {};
30 type = with types; attrsOf anything;
31 example = literalExpression ''
000032 '';
33- description = lib.mdDoc ''
34 Additional options and overrides for hbase-site.xml
35 <https://github.com/apache/hbase/blob/rel/2.4.11/hbase-common/src/main/resources/hbase-default.xml>
36 '';
···39 default = {};
40 type = with types; attrsOf anything;
41 internal = true;
42- description = lib.mdDoc ''
43 Internal option to add configs to hbase-site.xml based on module options
44 '';
45 };
···50 type = types.package;
51 default = pkgs.hbase;
52 defaultText = literalExpression "pkgs.hbase";
53- description = lib.mdDoc "HBase package";
54 };
5556 rootdir = mkOption {
57- description = lib.mdDoc ''
58 This option will set "hbase.rootdir" in hbase-site.xml and determine
59 the directory shared by region servers and into which HBase persists.
60 The URL should be 'fully-qualified' to include the filesystem scheme.
···68 default = "/hbase";
69 };
70 zookeeperQuorum = mkOption {
71- description = lib.mdDoc ''
72 This option will set "hbase.zookeeper.quorum" in hbase-site.xml.
73 Comma separated list of servers in the ZooKeeper ensemble.
74 '';
···76 example = "zk1.internal,zk2.internal,zk3.internal";
77 default = null;
78 };
79- master = {
80- enable = mkEnableOption (lib.mdDoc "HBase Master");
81- initHDFS = mkEnableOption (lib.mdDoc "initialization of the hbase directory on HDFS");
82-83- openFirewall = mkOption {
84- type = types.bool;
85- default = false;
86- description = lib.mdDoc ''
87- Open firewall ports for HBase master.
88- '';
89- };
90- };
91- regionServer = {
92- enable = mkEnableOption (lib.mdDoc "HBase RegionServer");
93-94- overrideHosts = mkOption {
95- type = types.bool;
96- default = true;
97- description = lib.mdDoc ''
98- Remove /etc/hosts entries for "127.0.0.2" and "::1" defined in nixos/modules/config/networking.nix
99- Regionservers must be able to resolve their hostnames to their IP addresses, through PTR records
100- or /etc/hosts entries.
101-102- '';
103 };
104-105- openFirewall = mkOption {
106- type = types.bool;
107- default = false;
108- description = lib.mdDoc ''
109- Open firewall ports for HBase master.
110- '';
111 };
112 };
113- };
114- };
115-116- config = mkMerge [
117- (mkIf cfg.hbase.master.enable {
118- services.hadoop.gatewayRole = {
119- enable = true;
120- enableHbaseCli = mkDefault true;
121- };
122-123- systemd.services.hbase-master = {
124- description = "HBase master";
125- wantedBy = [ "multi-user.target" ];
126-127- preStart = mkIf cfg.hbase.master.initHDFS ''
128- HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfsadmin -safemode wait
129- HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfs -mkdir -p ${cfg.hbase.rootdir}
130- HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfs -chown hbase ${cfg.hbase.rootdir}
131 '';
132-133- serviceConfig = {
134- User = "hbase";
135- SyslogIdentifier = "hbase-master";
136- ExecStart = "${cfg.hbase.package}/bin/hbase --config ${hadoopConf} " +
137- "master start";
138- Restart = "always";
139- };
140 };
0000141142- services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
143-144- networking.firewall.allowedTCPPorts = mkIf cfg.hbase.master.openFirewall [
145- 16000 16010
146- ];
147-148- })
149-150- (mkIf cfg.hbase.regionServer.enable {
151- services.hadoop.gatewayRole = {
152- enable = true;
153- enableHbaseCli = mkDefault true;
154- };
155-156- systemd.services.hbase-regionserver = {
157- description = "HBase RegionServer";
158- wantedBy = [ "multi-user.target" ];
159- serviceConfig = {
160- User = "hbase";
161- SyslogIdentifier = "hbase-regionserver";
162- ExecStart = "${cfg.hbase.package}/bin/hbase --config /etc/hadoop-conf/ " +
163- "regionserver start";
164- Restart = "always";
165- };
166- };
167-168- services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
169-170- networking = {
171- firewall.allowedTCPPorts = mkIf cfg.hbase.regionServer.openFirewall [
172- 16020 16030
173- ];
174- hosts = mkIf cfg.hbase.regionServer.overrideHosts {
175- "127.0.0.2" = mkForce [ ];
176- "::1" = mkForce [ ];
177- };
178- };
179- })
180181 (mkIf cfg.gatewayRole.enable {
182···192 isSystemUser = true;
193 };
194 })
195- ];
00000196}
···5 cfg = config.services.hadoop;
6 hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
7 mkIfNotNull = x: mkIf (x != null) x;
8+ # generic hbase role options
9+ hbaseRoleOption = name: extraOpts: {
10+ enable = mkEnableOption (mdDoc "HBase ${name}");
11+12+ openFirewall = mkOption {
13+ type = types.bool;
14+ default = false;
15+ description = mdDoc "Open firewall ports for HBase ${name}.";
16+ };
17+18+ restartIfChanged = mkOption {
19+ type = types.bool;
20+ default = false;
21+ description = mdDoc "Restart ${name} con config change.";
22+ };
23+24+ extraFlags = mkOption {
25+ type = with types; listOf str;
26+ default = [];
27+ example = literalExpression ''[ "--backup" ]'';
28+ description = mdDoc "Extra flags for the ${name} service.";
29+ };
30+31+ environment = mkOption {
32+ type = with types; attrsOf str;
33+ default = {};
34+ example = literalExpression ''
35+ {
36+ HBASE_MASTER_OPTS = "-Dcom.sun.management.jmxremote.ssl=true";
37+ }
38+ '';
39+ description = mdDoc "Environment variables passed to ${name}.";
40+ };
41+ } // extraOpts;
42+ # generic hbase role configs
43+ hbaseRoleConfig = name: ports: (mkIf cfg.hbase."${name}".enable {
44+ services.hadoop.gatewayRole = {
45+ enable = true;
46+ enableHbaseCli = mkDefault true;
47+ };
48+49+ systemd.services."hbase-${toLower name}" = {
50+ description = "HBase ${name}";
51+ wantedBy = [ "multi-user.target" ];
52+ path = with cfg; [ hbase.package ] ++ optional
53+ (with cfg.hbase.master; enable && initHDFS) package;
54+ preStart = mkIf (with cfg.hbase.master; enable && initHDFS)
55+ (concatStringsSep "\n" (
56+ map (x: "HADOOP_USER_NAME=hdfs hdfs --config /etc/hadoop-conf ${x}")[
57+ "dfsadmin -safemode wait"
58+ "dfs -mkdir -p ${cfg.hbase.rootdir}"
59+ "dfs -chown hbase ${cfg.hbase.rootdir}"
60+ ]
61+ ));
62+63+ inherit (cfg.hbase."${name}") environment;
64+ script = concatStringsSep " " (
65+ [
66+ "hbase --config /etc/hadoop-conf/"
67+ "${toLower name} start"
68+ ]
69+ ++ cfg.hbase."${name}".extraFlags
70+ ++ map (x: "--${toLower x} ${toString cfg.hbase.${name}.${x}}")
71+ (filter (x: hasAttr x cfg.hbase.${name}) ["port" "infoPort"])
72+ );
73+74+ serviceConfig = {
75+ User = "hbase";
76+ SyslogIdentifier = "hbase-${toLower name}";
77+ Restart = "always";
78+ };
79+ };
80+81+ services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
82+83+ networking = {
84+ firewall.allowedTCPPorts = mkIf cfg.hbase."${name}".openFirewall ports;
85+ hosts = mkIf (with cfg.hbase.regionServer; enable && overrideHosts) {
86+ "127.0.0.2" = mkForce [ ];
87+ "::1" = mkForce [ ];
88+ };
89+ };
90+91+ });
92in
93{
94 options.services.hadoop = {
9596+ gatewayRole.enableHbaseCli = mkEnableOption (mdDoc "HBase CLI tools");
9798 hbaseSiteDefault = mkOption {
99 default = {
···105 "hbase.cluster.distributed" = "true";
106 };
107 type = types.attrsOf types.anything;
108+ description = mdDoc ''
109 Default options for hbase-site.xml
110 '';
111 };
···113 default = {};
114 type = with types; attrsOf anything;
115 example = literalExpression ''
116+ {
117+ "hbase.hregion.max.filesize" = 20*1024*1024*1024;
118+ "hbase.table.normalization.enabled" = "true";
119+ }
120 '';
121+ description = mdDoc ''
122 Additional options and overrides for hbase-site.xml
123 <https://github.com/apache/hbase/blob/rel/2.4.11/hbase-common/src/main/resources/hbase-default.xml>
124 '';
···127 default = {};
128 type = with types; attrsOf anything;
129 internal = true;
130+ description = mdDoc ''
131 Internal option to add configs to hbase-site.xml based on module options
132 '';
133 };
···138 type = types.package;
139 default = pkgs.hbase;
140 defaultText = literalExpression "pkgs.hbase";
141+ description = mdDoc "HBase package";
142 };
143144 rootdir = mkOption {
145+ description = mdDoc ''
146 This option will set "hbase.rootdir" in hbase-site.xml and determine
147 the directory shared by region servers and into which HBase persists.
148 The URL should be 'fully-qualified' to include the filesystem scheme.
···156 default = "/hbase";
157 };
158 zookeeperQuorum = mkOption {
159+ description = mdDoc ''
160 This option will set "hbase.zookeeper.quorum" in hbase-site.xml.
161 Comma separated list of servers in the ZooKeeper ensemble.
162 '';
···164 example = "zk1.internal,zk2.internal,zk3.internal";
165 default = null;
166 };
167+ } // (let
168+ ports = port: infoPort: {
169+ port = mkOption {
170+ type = types.int;
171+ default = port;
172+ description = mdDoc "RPC port";
000000000000000000173 };
174+ infoPort = mkOption {
175+ type = types.int;
176+ default = infoPort;
177+ description = mdDoc "web UI port";
000178 };
179 };
180+ in mapAttrs hbaseRoleOption {
181+ master.initHDFS = mkEnableOption (mdDoc "initialization of the hbase directory on HDFS");
182+ regionServer.overrideHosts = mkOption {
183+ type = types.bool;
184+ default = true;
185+ description = mdDoc ''
186+ Remove /etc/hosts entries for "127.0.0.2" and "::1" defined in nixos/modules/config/networking.nix
187+ Regionservers must be able to resolve their hostnames to their IP addresses, through PTR records
188+ or /etc/hosts entries.
000000000189 '';
00000000190 };
191+ thrift = ports 9090 9095;
192+ rest = ports 8080 8085;
193+ });
194+ };
195196+ config = mkMerge ([
0000000000000000000000000000000000000197198 (mkIf cfg.gatewayRole.enable {
199···209 isSystemUser = true;
210 };
211 })
212+ ] ++ (mapAttrsToList hbaseRoleConfig {
213+ master = [ 16000 16010 ];
214+ regionServer = [ 16020 16030 ];
215+ thrift = with cfg.hbase.thrift; [ port infoPort ];
216+ rest = with cfg.hbase.rest; [ port infoPort ];
217+ }));
218}
···55echo "getting EC2 instance metadata..."
5657get_imds() {
58- # Intentionally no --fail here, so that we proceed even if e.g. a
59- # 404 was returned (but we still fail if we can't reach the IMDS
60- # server).
61- curl --silent --show-error --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@"
62}
6364get_imds -o "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
···55echo "getting EC2 instance metadata..."
5657get_imds() {
58+ # --fail to avoid populating missing files with 404 HTML response body
59+ # || true to allow the script to continue even when encountering a 404
60+ curl --silent --show-error --fail --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@" || true
061}
6263get_imds -o "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
···30 virtualisation.additionalPaths = [
31 pkgs.hello
32 pkgs.figlet
33-34- # This includes build dependencies all the way down. Not efficient,
35- # but we do need build deps to an *arbitrary* depth, which is hard to
36- # determine.
37- (allDrvOutputs nodes.server.config.system.build.toplevel)
38 ];
00039 };
40 server = { lib, ... }: {
41 imports = [ ./legacy/base-configuration.nix ];