jetbrains: add and improve tests

nixos/tests/all-tests.nix

···
   jellyfin = handleTest ./jellyfin.nix { };
   jenkins = handleTest ./jenkins.nix { };
   jenkins-cli = handleTest ./jenkins-cli.nix { };
+  jetbrains = import ./jetbrains/default.nix { inherit runTest; };
   jibri = handleTest ./jibri.nix { };
   jirafeau = handleTest ./jirafeau.nix { };
   jitsi-meet = handleTest ./jitsi-meet.nix { };

nixos/tests/jetbrains/default.nix

···
+{
+  runTest,
+}:
+{
+  plugins-available = runTest {
+    imports = [ ./plugins-available.nix ];
+  };
+}

nixos/tests/jetbrains/plugins-available.nix

···
+{ pkgs, ... }:
+
+# This test builds IDEA Community with some plugins and checks that they can be discovered by the IDE.
+let
+  idea = (
+    pkgs.jetbrains.plugins.addPlugins pkgs.jetbrains.idea-community-src [
+      # This is a "normal plugin", it's output must be linked into /idea-community/plugins.
+      "ideavim"
+      # This is a plugin where the output contains a single JAR file. This JAR file needs to be linked directly in /idea-community/plugins.
+      "wakatime"
+    ]
+  );
+in
+{
+  name = "jetbrains_build-ide-with-plugins";
+  meta.maintainers = pkgs.lib.teams.jetbrains.members;
+
+  nodes = {
+    server =
+      { config, pkgs, ... }:
+      {
+        environment.systemPackages = [
+          idea
+        ];
+      };
+  };
+
+  testScript =
+    { ... }:
+    ''
+      with subtest("ensure normal plugin is available"):
+          machine.succeed("find -L ${idea.out}/idea-community/plugins -type f -iname 'IdeaVIM-*.jar' | grep .")
+
+      with subtest("ensure single JAR file plugin is available"):
+          machine.succeed("""
+      PATH_TO_LINK=$(find ${idea.out}/idea-community/plugins -maxdepth 1 -type l -iname '*wakatime.jar' | grep .)
+      test -f $(readlink $PATH_TO_LINK)
+          """)
+    '';
+
+}

pkgs/applications/editors/jetbrains/bin/darwin.nix

···
   plugins ? [ ],
   buildNumber,
   ...
-}:
+}@args:
 
 let
   loname = lib.toLower productShort;
···
     ;
   passthru.buildNumber = buildNumber;
   passthru.product = product;
+  passthru.tests = args.passthru.tests;
   meta = meta // {
     mainProgram = loname;
   };

pkgs/applications/editors/jetbrains/bin/linux.nix

···
   extraLdPath ? [ ],
   extraWrapperArgs ? [ ],
   extraBuildInputs ? [ ],
+  ...
 }@args:
 
 let
···
   rec {
     inherit pname version src;
     passthru.buildNumber = buildNumber;
+    passthru.tests = args.passthru.tests;
     meta = args.meta // {
       mainProgram = pname;
     };

pkgs/applications/editors/jetbrains/default.nix

···
   stdenv,
   callPackage,
   fetchurl,
+  nixosTests,
 
   jdk,
   zlib,
···
       extraWrapperArgs ? [ ],
       extraLdPath ? [ ],
       extraBuildInputs ? [ ],
+      extraTests ? [ ],
     }:
     mkJetBrainsProductCore {
       inherit
···
       inherit (ideInfo."${pname}") wmClass product;
       productShort = ideInfo."${pname}".productShort or ideInfo."${pname}".product;
       meta = mkMeta ideInfo."${pname}".meta fromSource;
+      passthru.tests = extraTests // {
+        plugins = callPackage ./plugins/tests.nix { ideName = pname; };
+      };
       libdbm =
         if ideInfo."${pname}".meta.isOpenSource then
           communitySources."${pname}".libdbm
···
     pname = "idea-community";
     extraBuildInputs = [ stdenv.cc.cc ];
     fromSource = true;
+    extraTests = {
+      nixos-plugins-available = nixosTests.jetbrains.plugins-available;
+    };
   };
 
   idea-community =

pkgs/applications/editors/jetbrains/plugins/tests.nix

···
-{ jetbrains, writeText }:
+{
+  jetbrains,
+  symlinkJoin,
+  lib,
+  # If not set, all IDEs are tested.
+  ideName ? null,
+}:
+
+let
+
+  # Known broken plugins, PLEASE remove entries here whenever possible.
+  brokenPlugins = [
+    "github-copilot" # GitHub Copilot: https://github.com/NixOS/nixpkgs/issues/400317
+  ];
 
+  ideNames =
+    if ideName == null then
+      with jetbrains;
+      [
+        clion
+        datagrip
+        dataspell
+        goland
+        idea-community
+        idea-ultimate
+        mps
+        phpstorm
+        pycharm-community
+        pycharm-professional
+        rider
+        ruby-mine
+        rust-rover
+        webstorm
+      ]
+    else
+      [ (jetbrains.${ideName}) ];
+in
 {
   # Check to see if the process for adding plugins is breaking anything, instead of the plugins themselves
-  default =
+  empty =
     let
       modify-ide = ide: jetbrains.plugins.addPlugins ide [ ];
-      ides =
-        with jetbrains;
-        map modify-ide [
-          clion
-          datagrip
-          dataspell
-          goland
-          idea-community
-          idea-ultimate
-          mps
-          phpstorm
-          pycharm-community
-          pycharm-professional
-          rider
-          ruby-mine
-          rust-rover
-          webstorm
-        ];
-      paths = builtins.concatStringsSep " " ides;
     in
-    writeText "jb-ides" paths;
+    symlinkJoin {
+      name = "jetbrains-test-plugins-empty";
+      paths = (map modify-ide ideNames);
+    };
 
-  idea-ce-with-plugins = jetbrains.plugins.addPlugins jetbrains.idea-community [
-    "ideavim"
-    "nixidea"
-    # test JAR plugins
-    "wakatime"
-  ];
+  # Test all plugins. This will only build plugins compatible with the IDE and version. It will fail if the plugin is marked
+  # as compatible, but the build version is somehow not in the "builds" map (as that would indicate that something with update_plugins.py went wrong).
+  all =
+    let
+      pluginsJson = builtins.fromJSON (builtins.readFile ./plugins.json);
+      pluginsFor =
+        with lib.asserts;
+        ide:
+        builtins.map (plugin: plugin.name) (
+          builtins.filter (
+            plugin:
+            (
+              # Plugin has to not be broken
+              (!builtins.elem plugin.name brokenPlugins)
+              # IDE has to be compatible
+              && (builtins.elem ide.pname plugin.compatible)
+              # Assert: The build number needs to be included (if marked compatible)
+              && (assertMsg (builtins.elem ide.buildNumber (builtins.attrNames plugin.builds)) "For plugin ${plugin.name} no entry for IDE build ${ide.buildNumber} is defined, even though ${ide.pname} is on that build.")
+              # The plugin has to exist for the build
+              && (plugin.builds.${ide.buildNumber} != null)
+            )
+          ) (builtins.attrValues pluginsJson.plugins)
+        );
+      modify-ide = ide: jetbrains.plugins.addPlugins ide (pluginsFor ide);
+    in
+    symlinkJoin {
+      name = "jetbrains-test-plugins-all";
+      paths = (map modify-ide ideNames);
+    };
 }

pkgs/applications/editors/jetbrains/readme.md

···
 This directory contains the build expressions needed to build any of the jetbrains IDEs.
 The jdk is in `pkgs/development/compilers/jetbrains-jdk`.
-To test the build process of every IDE (as well as the process for adding plugins), build `jetbrains.plugins.tests.default`.
+
+## Tests:
+- To test the build process of every IDE (as well as the process for adding plugins), build `jetbrains.plugins.tests.empty`.
+- To test the build process with all plugins* supported by all IDEs, build `jetbrains.plugins.tests.all`. 
+- To test only plugins for a specific IDE*, build `jetbrains.ide-name.tests.plugins.all`.
+- To test that plugins are correctly stored in the plugins directory run the NixOS test `nixosTests.jetbrains.plugins-available`.
+
+*: Plugins marked as broken in nixpkgs are skipped: When updating/fixing plugins, please check the `brokenPlugins` in `plugins/tests.nix` and update it if needed.
 
 ## How to use plugins:
  - Get the ide you want and call `jetbrains.plugins.addPlugins` with a list of plugins you want to add.
  - The list of plugins can be a list of ids or names (as in `plugins/plugins.json`)
  - Example: `jetbrains.plugins.addPlugins jetbrains.pycharm-professional [ "nixidea" ]`
- - The list can also contain a drv giving a `.jar` or `.zip` (this is how you use a plugin not added to nixpkgs)
+ - The list can also contain drvs giving the directory contents of the plugin (this is how you use a plugin not added to nixpkgs) or a single `.jar` (executable). For an example, look at the implementation of `fetchPluginSrc` in `plugins/default.nix`.
 
 ### How to add a new plugin to nixpkgs
  - Find the page for the plugin on https://plugins.jetbrains.com