Merge pull request #30166 from LumiGuide/graphite-1.0.2

Fix graphite crash by upgrading from 0.9.15 -> 1.0.2

nixos/lib/test-driver/Machine.pm

···
     return $info;
 }
 
+# Fail if the given systemd unit is not in the "active" state.
+sub requireActiveUnit {
+    my ($self, $unit) = @_;
+    $self->nest("checking if unit ‘$unit’ has reached state 'active'", sub {
+        my $info = $self->getUnitInfo($unit);
+        my $state = $info->{ActiveState};
+        if ($state ne "active") {
+            die "Expected unit ‘$unit’ to to be in state 'active' but it is in state ‘$state’\n";
+        };
+    });
+}
 
 # Wait for a systemd unit to reach the "active" state.
 sub waitForUnit {

nixos/modules/services/monitoring/graphite.nix

···
   writeTextOrNull = f: t: mapNullable (pkgs.writeTextDir f) t;
 
   dataDir = cfg.dataDir;
+  staticDir = cfg.dataDir + "/static";
+
+  graphiteLocalSettingsDir = pkgs.runCommand "graphite_local_settings"
+    {inherit graphiteLocalSettings;} ''
+    mkdir -p $out
+    ln -s $graphiteLocalSettings $out/graphite_local_settings.py
+  '';
+
+  graphiteLocalSettings = pkgs.writeText "graphite_local_settings.py" (
+    "STATIC_ROOT = '${staticDir}'\n" +
+    optionalString (! isNull config.time.timeZone) "TIME_ZONE = '${config.time.timeZone}'\n"
+    + cfg.web.extraConfig
+  );
 
   graphiteApiConfig = pkgs.writeText "graphite-api.yaml" ''
     time_zone: ${config.time.timeZone}
···
         description = "Graphite web frontend port.";
         default = 8080;
         type = types.int;
+      };
+
+      extraConfig = mkOption {
+        type = types.str;
+        default = "";
+        description = ''
+          Graphite webapp settings. See:
+          <link xlink:href="http://graphite.readthedocs.io/en/latest/config-local-settings.html"/>
+        '';
       };
     };
 
···
                 ];
               };
               penvPack = "${penv}/${pkgs.python.sitePackages}";
-              # opt/graphite/webapp contains graphite/settings.py
-              # explicitly adding pycairo in path because it cannot be imported via buildEnv
-            in "${penvPack}/opt/graphite/webapp:${penvPack}:${pkgs.pythonPackages.pycairo}/${pkgs.python.sitePackages}";
+            in concatStringsSep ":" [
+                 "${graphiteLocalSettingsDir}"
+                 "${penvPack}/opt/graphite/webapp"
+                 "${penvPack}"
+                 # explicitly adding pycairo in path because it cannot be imported via buildEnv
+                 "${pkgs.pythonPackages.pycairo}/${pkgs.python.sitePackages}"
+               ];
           DJANGO_SETTINGS_MODULE = "graphite.settings";
           GRAPHITE_CONF_DIR = configDir;
           GRAPHITE_STORAGE_DIR = dataDir;
···
         };
         serviceConfig = {
           ExecStart = ''
-            ${pkgs.python27Packages.waitress}/bin/waitress-serve \
-            --host=${cfg.web.listenAddress} --port=${toString cfg.web.port} \
-            --call django.core.handlers.wsgi:WSGIHandler'';
+            ${pkgs.python27Packages.waitress-django}/bin/waitress-serve-django \
+              --host=${cfg.web.listenAddress} --port=${toString cfg.web.port}
+          '';
           User = "graphite";
           Group = "graphite";
           PermissionsStartOnly = true;
···
             mkdir -p ${dataDir}/{whisper/,log/webapp/}
             chmod 0700 ${dataDir}/{whisper/,log/webapp/}
 
-            # populate database
-            ${pkgs.python27Packages.graphite_web}/bin/manage-graphite.py syncdb --noinput
-
-            # create index
-            ${pkgs.python27Packages.graphite_web}/bin/build-index.sh
+            ${pkgs.pythonPackages.django_1_8}/bin/django-admin.py migrate --noinput
 
-            chown -R graphite:graphite ${cfg.dataDir}
+            chown -R graphite:graphite ${dataDir}
 
             touch ${dataDir}/db-created
+          fi
+
+          # Only collect static files when graphite_web changes.
+          if ! [ "${dataDir}/current_graphite_web" -ef "${pkgs.python27Packages.graphite_web}" ]; then
+            mkdir -p ${staticDir}
+            ${pkgs.pythonPackages.django_1_8}/bin/django-admin.py collectstatic  --noinput --clear
+            chown -R graphite:graphite ${staticDir}
+            ln -sfT "${pkgs.python27Packages.graphite_web}" "${dataDir}/current_graphite_web"
           fi
         '';
       };

nixos/release.nix

···
   tests.gnome3 = callTest tests/gnome3.nix {};
   tests.gnome3-gdm = callTest tests/gnome3-gdm.nix {};
   tests.grafama = callTest tests/grafana.nix {};
+  tests.graphite = callTest tests/graphite.nix {};
   tests.hardened = callTest tests/hardened.nix { };
   tests.hibernate = callTest tests/hibernate.nix {};
   tests.hound = callTest tests/hound.nix {};

nixos/tests/graphite.nix

···
+import ./make-test.nix ({ pkgs, ...} :
+{
+  name = "graphite";
+  nodes = {
+    one =
+      { config, pkgs, ... }: {
+        services.graphite = {
+          web = {
+            enable = true;
+          };
+          carbon = {
+            enableCache = true;
+          };
+        };
+      };
+    };
+
+  testScript = ''
+    startAll;
+    $one->waitForUnit("default.target");
+    $one->requireActiveUnit("graphiteWeb.service");
+    $one->requireActiveUnit("carbonCache.service");
+    $one->succeed("echo \"foo 1 `date +%s`\" | nc -q0 localhost 2003");
+    $one->waitUntilSucceeds("curl 'http://localhost:8080/metrics/find/?query=foo&format=treejson' --silent | grep foo")
+  '';
+})

pkgs/development/python-modules/waitress-django/default.nix

···
+{ buildPythonPackage, django_1_8, waitress }:
+buildPythonPackage {
+  name = "waitress-django";
+  src = ./.;
+  pythonPath = [ django_1_8 waitress ];
+  doCheck = false;
+  meta.description = "A waitress WSGI server serving django";
+}

pkgs/development/python-modules/waitress-django/setup.py

···
+#!/usr/bin/env python
+
+from distutils.core import setup
+
+setup( name         = "waitress-django"
+     , version      = "0.0.0"
+     , description  = "A waitress WSGI server serving django"
+     , author       = "Bas van Dijk"
+     , author_email = "[e-mail]"
+     , package_dir  = {"" : "src"}
+     , scripts      = ["src/waitress-serve-django"]
+     )

pkgs/development/python-modules/waitress-django/src/waitress-serve-django

···
+#!/usr/bin/env python
+import sys
+from waitress import serve
+from waitress.adjustments import Adjustments
+import django
+from django.core.handlers.wsgi import WSGIHandler
+from django.contrib.staticfiles.handlers import StaticFilesHandler
+
+if __name__ == "__main__":
+  kw, args = Adjustments.parse_args(sys.argv[1:])
+  django.setup()
+  # These arguments are specific to the runner, not waitress itself.
+  del kw['call'], kw['help']
+  serve(StaticFilesHandler(WSGIHandler()), **kw)

pkgs/top-level/python-packages.nix

···
 
   buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args );
 
-  graphiteVersion = "0.9.15";
+  graphiteVersion = "1.0.2";
 
   fetchPypi = makeOverridable( {format ? "setuptools", ... } @attrs:
     let
···
 
   django_tagging = callPackage ../development/python-modules/django_tagging { };
 
-  django_tagging_0_3 = self.django_tagging.overrideAttrs (attrs: rec {
-    name = "django-tagging-0.3.6";
+  django_tagging_0_4_3 = self.django_tagging.overrideAttrs (attrs: rec {
+    name = "django-tagging-0.4.3";
 
     src = pkgs.fetchurl {
       url = "mirror://pypi/d/django-tagging/${name}.tar.gz";
-      sha256 = "03zlbq13rydfh28wh0jk3x3cjk9x6jjmqnx1i3ngjmfwbxf8x6j1";
+      sha256 = "0617azpmp6jpg3d88v2ir97qrc9aqcs2s9gyvv9bgf2cp55khxhs";
     };
-    propagatedBuildInputs = with self; [ django ];
+    propagatedBuildInputs = with self; [ django_1_8 ];
   });
 
   django_classytags = buildPythonPackage rec {
···
   };
 
   waitress = buildPythonPackage rec {
-    name = "waitress-0.8.9";
+    name = "waitress-1.0.2";
 
     src = pkgs.fetchurl {
       url = "mirror://pypi/w/waitress/${name}.tar.gz";
-      sha256 = "826527dc9d334ed4ed76cdae672fdcbbccf614186657db71679ab58df869458a";
+      sha256 = "0pw6yyxi348r2xpq3ykqnf7gwi881azv2422d2ixb0xi5jws2ky7";
     };
 
     doCheck = false;
···
        platforms = platforms.all;
     };
   };
+
+  waitress-django = callPackage ../development/python-modules/waitress-django { };
 
   webassets = buildPythonPackage rec {
     name = "webassets-${version}";
···
 
     src = pkgs.fetchurl {
       url = "mirror://pypi/w/whisper/${name}.tar.gz";
-      sha256 = "1chkphxwnwvy2cs7jc2h2i0lqqvi9jx6vqj3ly88lwk7m35r4ss2";
+      sha256 = "1v1bi3fl1i6p4z4ki692bykrkw6907dn3mfq0151f70lvi3zpns3";
     };
 
     # error: invalid command 'test'
···
 
     src = pkgs.fetchurl {
       url = "mirror://pypi/c/carbon/${name}.tar.gz";
-      sha256 = "f01db6d37726c6fc0a8aaa66a7bf14436b0dd0d62ef3c20ecb31605a4d365d2e";
+      sha256 = "142smpmgbnjinvfb6s4ijazish4vfgzyd8zcmdkh55y051fkixkn";
     };
 
     propagatedBuildInputs = with self; [ whisper txamqp zope_interface twisted ];
···
 
     src = pkgs.fetchurl rec {
       url = "mirror://pypi/g/graphite-web/${name}.tar.gz";
-      sha256 = "1c0kclbv8shv9nvjx19wqm4asia58s3qmd9fapchc6y9fjpjax6q";
+      sha256 = "0q8bwlj75jqyzmazfsi5sa26xl58ssa8wdxm2l4j0jqyn8xpfnmc";
     };
 
-    propagatedBuildInputs = with self; [ django django_tagging_0_3 whisper pycairo ldap memcached pytz ];
+    propagatedBuildInputs = with self; [
+      django_1_8 django_tagging_0_4_3 whisper pycairo cairocffi
+      ldap memcached pytz urllib3 scandir
+    ];
 
     postInstall = ''
       wrapProgram $out/bin/run-graphite-devel-server.py \
···
     '';
 
     preConfigure = ''
-      substituteInPlace webapp/graphite/thirdparty/pytz/__init__.py --replace '/usr/share/zoneinfo' '/etc/zoneinfo'
-      substituteInPlace webapp/graphite/settings.py --replace "join(WEBAPP_DIR, 'content')" "join('$out', 'webapp', 'content')"
-      cp webapp/graphite/manage.py bin/manage-graphite.py
-      substituteInPlace bin/manage-graphite.py --replace 'settings' 'graphite.settings'
+      # graphite is configured by storing a local_settings.py file inside the
+      # graphite python package. Since that package is stored in the immutable
+      # Nix store we can't modify it. So how do we configure graphite?
+      #
+      # First of all we rename "graphite.local_settings" to
+      # "graphite_local_settings" so that the settings are not looked up in the
+      # graphite package anymore. Secondly we place a directory containing a
+      # graphite_local_settings.py on the PYTHONPATH in the graphite module
+      # <nixpkgs/nixos/modules/services/monitoring/graphite.nix>.
+      substituteInPlace webapp/graphite/settings.py \
+        --replace "graphite.local_settings" " graphite_local_settings"
+
+      substituteInPlace webapp/graphite/settings.py \
+        --replace "join(WEBAPP_DIR, 'content')" "join('$out', 'webapp', 'content')"
     '';
 
     # error: invalid command 'test'