Merge pull request #61312 from Yarny0/tsm-client

TSM client

nixos/modules/module-list.nix

···
   ./programs/sway.nix
   ./programs/thefuck.nix
   ./programs/tmux.nix
+  ./programs/tsm-client.nix
   ./programs/udevil.nix
   ./programs/venus.nix
   ./programs/vim.nix
···
   ./services/backup/restic-rest-server.nix
   ./services/backup/rsnapshot.nix
   ./services/backup/tarsnap.nix
+  ./services/backup/tsm.nix
   ./services/backup/znapzend.nix
   ./services/cluster/hadoop/default.nix
   ./services/cluster/kubernetes/addons/dns.nix

nixos/modules/programs/tsm-client.nix

···
+{ config, lib, pkgs, ... }:
+
+let
+
+  inherit (builtins) length map;
+  inherit (lib.attrsets) attrNames filterAttrs hasAttr mapAttrs mapAttrsToList optionalAttrs;
+  inherit (lib.modules) mkDefault mkIf;
+  inherit (lib.options) literalExample mkEnableOption mkOption;
+  inherit (lib.strings) concatStringsSep optionalString toLower;
+  inherit (lib.types) addCheck attrsOf lines loaOf nullOr package path port str strMatching submodule;
+
+  # Checks if given list of strings contains unique
+  # elements when compared without considering case.
+  # Type: checkIUnique :: [string] -> bool
+  # Example: checkIUnique ["foo" "Foo"] => false
+  checkIUnique = lst:
+    let
+      lenUniq = l: length (lib.lists.unique l);
+    in
+      lenUniq lst == lenUniq (map toLower lst);
+
+  # TSM rejects servername strings longer than 64 chars.
+  servernameType = strMatching ".{1,64}";
+
+  serverOptions = { name, config, ... }: {
+    options.name = mkOption {
+      type = servernameType;
+      example = "mainTsmServer";
+      description = ''
+        Local name of the IBM TSM server,
+        must be uncapitalized and no longer than 64 chars.
+        The value will be used for the
+        <literal>server</literal>
+        directive in <filename>dsm.sys</filename>.
+      '';
+    };
+    options.server = mkOption {
+      type = strMatching ".+";
+      example = "tsmserver.company.com";
+      description = ''
+        Host/domain name or IP address of the IBM TSM server.
+        The value will be used for the
+        <literal>tcpserveraddress</literal>
+        directive in <filename>dsm.sys</filename>.
+      '';
+    };
+    options.port = mkOption {
+      type = addCheck port (p: p<=32767);
+      default = 1500;  # official default
+      description = ''
+        TCP port of the IBM TSM server.
+        The value will be used for the
+        <literal>tcpport</literal>
+        directive in <filename>dsm.sys</filename>.
+        TSM does not support ports above 32767.
+      '';
+    };
+    options.node = mkOption {
+      type = strMatching ".+";
+      example = "MY-TSM-NODE";
+      description = ''
+        Target node name on the IBM TSM server.
+        The value will be used for the
+        <literal>nodename</literal>
+        directive in <filename>dsm.sys</filename>.
+      '';
+    };
+    options.genPasswd = mkEnableOption ''
+      automatic client password generation.
+      This option influences the
+      <literal>passwordaccess</literal>
+      directive in <filename>dsm.sys</filename>.
+      The password will be stored in the directory
+      given by the option <option>passwdDir</option>.
+      <emphasis>Caution</emphasis>:
+      If this option is enabled and the server forces
+      to renew the password (e.g. on first connection),
+      a random password will be generated and stored
+    '';
+    options.passwdDir = mkOption {
+      type = path;
+      example = "/home/alice/tsm-password";
+      description = ''
+        Directory that holds the TSM
+        node's password information.
+        The value will be used for the
+        <literal>passworddir</literal>
+        directive in <filename>dsm.sys</filename>.
+      '';
+    };
+    options.includeExclude = mkOption {
+      type = lines;
+      default = "";
+      example = ''
+        exclude.dir     /nix/store
+        include.encrypt /home/.../*
+      '';
+      description = ''
+        <literal>include.*</literal> and
+        <literal>exclude.*</literal> directives to be
+        used when sending files to the IBM TSM server.
+        The lines will be written into a file that the
+        <literal>inclexcl</literal>
+        directive in <filename>dsm.sys</filename> points to.
+      '';
+    };
+    options.extraConfig = mkOption {
+      # TSM option keys are case insensitive;
+      # we have to ensure there are no keys that
+      # differ only by upper and lower case.
+      type = addCheck
+        (attrsOf (nullOr str))
+        (attrs: checkIUnique (attrNames attrs));
+      default = {};
+      example.compression = "yes";
+      example.passwordaccess = null;
+      description = ''
+        Additional key-value pairs for the server stanza.
+        Values must be strings, or <literal>null</literal>
+        for the key not to be used in the stanza
+        (e.g. to overrule values generated by other options).
+      '';
+    };
+    options.text = mkOption {
+      type = lines;
+      example = literalExample
+        ''lib.modules.mkAfter "compression no"'';
+      description = ''
+        Additional text lines for the server stanza.
+        This option can be used if certion configuration keys
+        must be used multiple times or ordered in a certain way
+        as the <option>extraConfig</option> option can't
+        control the order of lines in the resulting stanza.
+        Note that the <literal>server</literal>
+        line at the beginning of the stanza is
+        not part of this option's value.
+      '';
+    };
+    options.stanza = mkOption {
+      type = str;
+      internal = true;
+      visible = false;
+      description = "Server stanza text generated from the options.";
+    };
+    config.name = mkDefault name;
+    # Client system-options file directives are explained here:
+    # https://www.ibm.com/support/knowledgecenter/SSEQVQ_8.1.8/client/c_opt_usingopts.html
+    config.extraConfig =
+      mapAttrs (lib.trivial.const mkDefault) (
+        {
+          commmethod = "v6tcpip";  # uses v4 or v6, based on dns lookup result
+          tcpserveraddress = config.server;
+          tcpport = builtins.toString config.port;
+          nodename = config.node;
+          passwordaccess = if config.genPasswd then "generate" else "prompt";
+          passworddir = ''"${config.passwdDir}"'';
+        } // optionalAttrs (config.includeExclude!="") {
+          inclexcl = ''"${pkgs.writeText "inclexcl.dsm.sys" config.includeExclude}"'';
+        }
+      );
+    config.text =
+      let
+        attrset = filterAttrs (k: v: v!=null) config.extraConfig;
+        mkLine = k: v: k + optionalString (v!="") "  ${v}";
+        lines = mapAttrsToList mkLine attrset;
+      in
+        concatStringsSep "\n" lines;
+    config.stanza = ''
+      server  ${config.name}
+      ${config.text}
+    '';
+  };
+
+  options.programs.tsmClient = {
+    enable = mkEnableOption ''
+      IBM Spectrum Protect (Tivoli Storage Manager, TSM)
+      client command line applications with a
+      client system-options file "dsm.sys"
+    '';
+    servers = mkOption {
+      type = loaOf (submodule [ serverOptions ]);
+      default = {};
+      example.mainTsmServer = {
+        server = "tsmserver.company.com";
+        node = "MY-TSM-NODE";
+        extraConfig.compression = "yes";
+      };
+      description = ''
+        Server definitions ("stanzas")
+        for the client system-options file.
+      '';
+    };
+    defaultServername = mkOption {
+      type = nullOr servernameType;
+      default = null;
+      example = "mainTsmServer";
+      description = ''
+        If multiple server stanzas are declared with
+        <option>programs.tsmClient.servers</option>,
+        this option may be used to name a default
+        server stanza that IBM TSM uses in the absence of
+        a user-defined <filename>dsm.opt</filename> file.
+        This option translates to a
+        <literal>defaultserver</literal> configuration line.
+      '';
+    };
+    dsmSysText = mkOption {
+      type = lines;
+      readOnly = true;
+      description = ''
+        This configuration key contains the effective text
+        of the client system-options file "dsm.sys".
+        It should not be changed, but may be
+        used to feed the configuration into other
+        TSM-depending packages used on the system.
+      '';
+    };
+    package = mkOption {
+      type = package;
+      default = pkgs.tsm-client;
+      defaultText = "pkgs.tsm-client";
+      example = literalExample "pkgs.tsm-client-withGui";
+      description = ''
+        The TSM client derivation to be
+        added to the system environment.
+        It will called with <literal>.override</literal>
+        to add paths to the client system-options file.
+      '';
+    };
+    wrappedPackage = mkOption {
+      type = package;
+      readOnly = true;
+      description = ''
+        The TSM client derivation, wrapped with the path
+        to the client system-options file "dsm.sys".
+        This option is to provide the effective derivation
+        for other modules that want to call TSM executables.
+      '';
+    };
+  };
+
+  cfg = config.programs.tsmClient;
+
+  assertions = [
+    {
+      assertion = checkIUnique (mapAttrsToList (k: v: v.name) cfg.servers);
+      message = ''
+        TSM servernames contain duplicate name
+        (note that case doesn't matter!)
+      '';
+    }
+    {
+      assertion = (cfg.defaultServername!=null)->(hasAttr cfg.defaultServername cfg.servers);
+      message = "TSM defaultServername not found in list of servers";
+    }
+  ];
+
+  dsmSysText = ''
+    ****  IBM Spectrum Protect (Tivoli Storage Manager)
+    ****  client system-options file "dsm.sys".
+    ****  Do not edit!
+    ****  This file is generated by NixOS configuration.
+
+    ${optionalString (cfg.defaultServername!=null) "defaultserver  ${cfg.defaultServername}"}
+
+    ${concatStringsSep "\n" (mapAttrsToList (k: v: v.stanza) cfg.servers)}
+  '';
+
+in
+
+{
+
+  inherit options;
+
+  config = mkIf cfg.enable {
+    inherit assertions;
+    programs.tsmClient.dsmSysText = dsmSysText;
+    programs.tsmClient.wrappedPackage = cfg.package.override rec {
+      dsmSysCli = pkgs.writeText "dsm.sys" cfg.dsmSysText;
+      dsmSysApi = dsmSysCli;
+    };
+    environment.systemPackages = [ cfg.wrappedPackage ];
+  };
+
+  meta.maintainers = [ lib.maintainers.yarny ];
+
+}

nixos/modules/services/backup/tsm.nix

···
+{ config, lib, ... }:
+
+let
+
+  inherit (lib.attrsets) hasAttr;
+  inherit (lib.modules) mkDefault mkIf;
+  inherit (lib.options) mkEnableOption mkOption;
+  inherit (lib.types) nullOr strMatching;
+
+  options.services.tsmBackup = {
+    enable = mkEnableOption ''
+      automatic backups with the
+      IBM Spectrum Protect (Tivoli Storage Manager, TSM) client.
+      This also enables
+      <option>programs.tsmClient.enable</option>
+    '';
+    command = mkOption {
+      type = strMatching ".+";
+      default = "backup";
+      example = "incr";
+      description = ''
+        The actual command passed to the
+        <literal>dsmc</literal> executable to start the backup.
+      '';
+    };
+    servername = mkOption {
+      type = strMatching ".+";
+      example = "mainTsmServer";
+      description = ''
+        Create a systemd system service
+        <literal>tsm-backup.service</literal> that starts
+        a backup based on the given servername's stanza.
+        Note that this server's
+        <option>passwdDir</option> will default to
+        <filename>/var/lib/tsm-backup/password</filename>
+        (but may be overridden);
+        also, the service will use
+        <filename>/var/lib/tsm-backup</filename> as
+        <literal>HOME</literal> when calling
+        <literal>dsmc</literal>.
+      '';
+    };
+    autoTime = mkOption {
+      type = nullOr (strMatching ".+");
+      default = null;
+      example = "12:00";
+      description = ''
+        The backup service will be invoked
+        automatically at the given date/time,
+        which must be in the format described in
+        <citerefentry><refentrytitle>systemd.time</refentrytitle><manvolnum>5</manvolnum></citerefentry>.
+        The default <literal>null</literal>
+        disables automatic backups.
+      '';
+    };
+  };
+
+  cfg = config.services.tsmBackup;
+  cfgPrg = config.programs.tsmClient;
+
+  assertions = [
+    {
+      assertion = hasAttr cfg.servername cfgPrg.servers;
+      message = "TSM service servername not found in list of servers";
+    }
+    {
+      assertion = cfgPrg.servers.${cfg.servername}.genPasswd;
+      message = "TSM service requires automatic password generation";
+    }
+  ];
+
+in
+
+{
+
+  inherit options;
+
+  config = mkIf cfg.enable {
+    inherit assertions;
+    programs.tsmClient.enable = true;
+    programs.tsmClient.servers."${cfg.servername}".passwdDir =
+      mkDefault "/var/lib/tsm-backup/password";
+    systemd.services.tsm-backup = {
+      description = "IBM Spectrum Protect (Tivoli Storage Manager) Backup";
+      # DSM_LOG needs a trailing slash to have it treated as a directory.
+      # `/var/log` would be littered with TSM log files otherwise.
+      environment.DSM_LOG = "/var/log/tsm-backup/";
+      # TSM needs a HOME dir to store certificates.
+      environment.HOME = "/var/lib/tsm-backup";
+      # for exit status description see
+      # https://www.ibm.com/support/knowledgecenter/en/SSEQVQ_8.1.8/client/c_sched_rtncode.html
+      serviceConfig.SuccessExitStatus = "4 8";
+      # The `-se` option must come after the command.
+      # The `-optfile` option suppresses a `dsm.opt`-not-found warning.
+      serviceConfig.ExecStart =
+        "${cfgPrg.wrappedPackage}/bin/dsmc ${cfg.command} -se='${cfg.servername}' -optfile=/dev/null";
+      serviceConfig.LogsDirectory = "tsm-backup";
+      serviceConfig.StateDirectory = "tsm-backup";
+      serviceConfig.StateDirectoryMode = "0750";
+      startAt = mkIf (cfg.autoTime!=null) cfg.autoTime;
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.yarny ];
+
+}

pkgs/tools/backup/tsm-client/default.nix

···
+{ lib
+, stdenv
+, autoPatchelfHook
+, buildEnv
+, fetchurl
+, makeWrapper
+, procps
+, zlib
+# optional packages that enable certain features
+, acl ? null  # EXT2/EXT3/XFS ACL support
+, jdk8 ? null  # Java GUI
+, lvm2 ? null  # LVM image backup and restore functions
+# path to `dsm.sys` configuration files
+, dsmSysCli ? "/etc/tsm-client/cli.dsm.sys"
+, dsmSysApi ? "/etc/tsm-client/api.dsm.sys"
+}:
+
+
+# For an explanation of optional packages
+# (features provided by them, version limits), see
+# https://www-01.ibm.com/support/docview.wss?uid=swg21052223#Version%208.1
+
+
+# IBM Tivoli Storage Manager Client uses a system-wide
+# client system-options file `dsm.sys` and expects it
+# to be located in a directory within the package.
+# Note that the command line client and the API use
+# different "dms.sys" files (located in different directories).
+# Since these files contain settings to be altered by the
+# admin user (e.g. TSM server name), we create symlinks
+# in place of the files that the client attempts to open.
+# Use the arguments `dsmSysCli` and `dsmSysApi` to
+# provide the location of the configuration files for
+# the command-line interface and the API, respectively.
+#
+# While the command-line interface contains wrappers
+# that help the executables find the configuration file,
+# packages that link against the API have to
+# set the environment variable `DSMI_DIR` to
+# point to this derivations `/dsmi_dir` directory symlink.
+# Other environment variables might be necessary,
+# depending on local configuration or usage; see:
+# https://www.ibm.com/support/knowledgecenter/en/SSEQVQ_8.1.8/client/c_cfg_sapiunix.html
+
+
+# The newest version of TSM client should be discoverable
+# by going the the `downloadPage` (see `meta` below),
+# there to "Client Latest Downloads",
+# "IBM Spectrum Protect Client Downloads and READMEs",
+# then to "Linux x86_64 Ubuntu client" (as of 2019-07-15).
+
+
+let
+
+  meta = {
+    homepage = https://www.ibm.com/us-en/marketplace/data-protection-and-recovery;
+    downloadPage = https://www-01.ibm.com/support/docview.wss?uid=swg21239415;
+    platforms = [ "x86_64-linux" ];
+    license = lib.licenses.unfree;
+    maintainers = [ lib.maintainers.yarny ];
+    description = "IBM Spectrum Protect (Tivoli Storage Manager) CLI and API";
+    longDescription = ''
+      IBM Spectrum Protect (Tivoli Storage Manager) provides
+      a single point of control for backup and recovery.
+      This package contains the client software, that is,
+      a command line client and linkable libraries.
+
+      Note that the software requires a system-wide
+      client system-options file (commonly named "dsm.sys").
+      This package allows to use separate files for
+      the command-line interface and for the linkable API.
+      The location of those files can
+      be provided as build parameters.
+    '';
+  };
+
+  unwrapped = stdenv.mkDerivation rec {
+    name = "tsm-client-${version}-unwrapped";
+    version = "8.1.8.0";
+    src = fetchurl {
+      url = "ftp://public.dhe.ibm.com/storage/tivoli-storage-management/maintenance/client/v8r1/Linux/LinuxX86_DEB/BA/v818/${version}-TIV-TSMBAC-LinuxX86_DEB.tar";
+      sha256 = "0c1d0jm0i7qjd314nhj2vj8fs7sncm1x2n4d6dg4049jniyvjhpk";
+    };
+    inherit meta;
+
+    nativeBuildInputs = [
+      autoPatchelfHook
+    ];
+    buildInputs = [
+      stdenv.cc.cc
+      zlib
+    ];
+    runtimeDependencies = [
+      lvm2
+    ];
+    sourceRoot = ".";
+
+    postUnpack = ''
+      for debfile in *.deb
+      do
+        ar -x "$debfile"
+        tar --xz --extract --file=data.tar.xz
+        rm data.tar.xz
+      done
+    '';
+
+    installPhase = ''
+      runHook preInstall
+      mkdir --parents $out
+      mv --target-directory=$out usr/* opt
+      runHook postInstall
+    '';
+
+    # Fix relative symlinks after `/usr` was moved up one level
+    preFixup = ''
+      for link in $out/lib/* $out/bin/*
+      do
+        target=$(readlink "$link")
+        if [ "$(cut -b -6 <<< "$target")" != "../../" ]
+        then
+          echo "cannot fix this symlink: $link -> $target"
+          exit 1
+        fi
+        ln --symbolic --force --no-target-directory "$out/$(cut -b 7- <<< "$target")" "$link"
+      done
+    '';
+  };
+
+in
+
+buildEnv {
+  name = "tsm-client-${unwrapped.version}";
+  inherit meta;
+  passthru = { inherit unwrapped; };
+  paths = [ unwrapped ];
+  buildInputs = [ makeWrapper ];
+  pathsToLink = [
+    "/"
+    "/bin"
+    "/opt/tivoli/tsm/client/ba/bin"
+    "/opt/tivoli/tsm/client/api/bin64"
+  ];
+  # * Provide top-level symlinks `dsm_dir` and `dsmi_dir`
+  #   to the so-called "installation directories"
+  # * Add symlinks to the "installation directories"
+  #   that point to the `dsm.sys` configuration files
+  # * Drop the Java GUI executable unless `jdk` is present
+  # * Create wrappers for the command-line interface to
+  #   prepare `PATH` and `DSM_DIR` environment variables
+  postBuild = ''
+    ln --symbolic --no-target-directory opt/tivoli/tsm/client/ba/bin $out/dsm_dir
+    ln --symbolic --no-target-directory opt/tivoli/tsm/client/api/bin64 $out/dsmi_dir
+    ln --symbolic --no-target-directory "${dsmSysCli}" $out/dsm_dir/dsm.sys
+    ln --symbolic --no-target-directory "${dsmSysApi}" $out/dsmi_dir/dsm.sys
+    ${lib.strings.optionalString (jdk8==null) "rm $out/bin/dsmj"}
+    for bin in $out/bin/*
+    do
+      target=$(readlink "$bin")
+      rm "$bin"
+      makeWrapper "$target" "$bin" \
+        --prefix PATH : "$out/dsm_dir:${lib.strings.makeBinPath [ procps acl jdk8 ]}" \
+        --set DSM_DIR $out/dsm_dir
+    done
+  '';
+}

pkgs/top-level/all-packages.nix

···
 
   teamocil = callPackage ../tools/misc/teamocil { };
 
+  tsm-client = callPackage ../tools/backup/tsm-client { jdk8 = null; };
+  tsm-client-withGui = callPackage ../tools/backup/tsm-client { };
+
   tridactyl-native = callPackage ../tools/networking/tridactyl-native { };
 
   trompeloeil = callPackage ../development/libraries/trompeloeil { };