Merge pull request #31333 from ElvishJerricco/unextracted-all-cabal-hashes

Unextracted all cabal hashes

pkgs/data/misc/hackage/default.nix

···
-{ fetchFromGitHub
-, stdenv
-, pkgconfig
-, libarchive
-, glib
-, # Override this to use a different revision
-  src-spec ?
-    { owner = "commercialhaskell";
-      repo = "all-cabal-hashes";
-      rev = "5e87c40f2cd96bd5dd953758e82f302107c7895e";
-      sha256 = "12rw5fld64s0a2zjsdijfs0dv6vc6z7gcf24h4m2dmymzms4namg";
-    }
-, lib
-}:
+{ fetchurl }:
 
-# Use builtins.fetchTarball "https://github.com/commercialhaskell/all-cabal-hashes/archive/hackage.tar.gz"
-# instead if you want the latest Hackage automatically at the price of frequent re-downloads.
-let partition-all-cabal-hashes = stdenv.mkDerivation
-      { name = "partition-all-cabal-hashes";
-        src = ./partition-all-cabal-hashes.c;
-        unpackPhase = "true";
-        buildInputs = [ libarchive glib ];
-        nativeBuildInputs = [ pkgconfig ];
-        buildPhase =
-          "cc -O3 $(pkg-config --cflags --libs libarchive glib-2.0) $src -o partition-all-cabal-hashes";
-        installPhase =
-          ''
-            mkdir -p $out/bin
-            install -m755 partition-all-cabal-hashes $out/bin
-          '';
-      };
-in fetchFromGitHub (src-spec //
-  { postFetch = "${partition-all-cabal-hashes}/bin/partition-all-cabal-hashes $downloadedFile $out";
-  })
+fetchurl {
+  url = "https://github.com/commercialhaskell/all-cabal-hashes/archive/5e87c40f2cd96bd5dd953758e82f302107c7895e.tar.gz";
+  sha256 = "0hjkddda9mdm21nb9bkhr9n5r9jllisif1qmzha91a9cps5w1mx5";
+}

pkgs/data/misc/hackage/partition-all-cabal-hashes.c

···
-#include <stdio.h>
-#include <ctype.h>
-#include <glib.h>
-#include <string.h>
-#include <locale.h>
-#include <archive.h>
-#include <archive_entry.h>
-
-static char * case_normalize(char * str) {
-  for (char * iter = str; *iter; ++iter) {
-    *iter = tolower(*iter);
-  }
-  return str;
-}
-
-static gint compare_str(const void * a, const void * b, void * _) {
-  return strcmp(a, b);
-}
-
-int main(int argc, char ** argv) {
-  if (argc != 3) {
-    fprintf(stderr, "Usage: %s TARBALL OUTPUT\n", argv[0]);
-    return 1;
-  }
-
-  size_t output_len = strlen(argv[2]);
-
-  /* Switch to standard locale to ensure consistency in case-folding.
-   */
-  setlocale(LC_CTYPE, "C");
-
-  /* Map from case-normalized package name to a sorted sequence of
-   * package names in the equivalence class defined by
-   * case-normalization.
-   */
-  GHashTable * equivalence_classes =
-    g_hash_table_new(g_str_hash, g_str_equal);
-
-  /* Open up the tarball.
-   */
-  struct archive * ar = archive_read_new();
-  if (!ar) {
-    perror("Allocating archive structure");
-    return 1;
-  }
-  archive_read_support_filter_gzip(ar);
-  archive_read_support_format_tar(ar);
-  if (archive_read_open_filename( ar
-                                , argv[1]
-                                , 10240
-                                ) == ARCHIVE_FATAL) {
-    fprintf( stderr
-           , "Error opening %s: %s\n"
-           , argv[0]
-           , archive_error_string(ar)
-           );
-    return 1;
-  }
-
-  /* Extract the length of the output directory that prefixes all
-   * tarball entries from the first entry in the tarball.
-   */
-  struct archive_entry * ent;
-  int err = archive_read_next_header(ar, &ent);
-  if (err != ARCHIVE_OK) {
-    if (err == ARCHIVE_EOF) {
-      fprintf( stderr
-             , "No entries in %s, surely this is an error!\n"
-             , argv[1]
-             );
-    } else {
-      fprintf( stderr
-             , "Error reading entry from %s: %s\n"
-             , argv[1]
-             , archive_error_string(ar)
-             );
-    }
-    return 1;
-  }
-  const char * path = archive_entry_pathname(ent);
-  /* Number of characters from the start of the path name until after
-   * the slash after the leading directory.
-   */
-  size_t prefix_len = strchr(path, '/') - path + 1;
-
-  /* Extract each entry to the right partition.
-   */
-  do {
-    path = archive_entry_pathname(ent) + prefix_len;
-    const char * pkg_end = strchr(path, '/');
-    if (!pkg_end)
-      /* If there is no second slash, then this is either just the entry
-       * corresponding to the root or some non-package file (e.g.
-       * travis.yml). In either case, we don't care.
-       */
-      continue;
-
-    /* Find our package in the equivalence class map.
-     */
-    char * pkg_name = g_strndup(path, pkg_end - path);
-    char * pkg_normalized =
-      case_normalize(g_strndup(path, pkg_end - path));
-    GSequence * pkg_class =
-      g_hash_table_lookup(equivalence_classes, pkg_normalized);
-    gint partition_num;
-    if (!pkg_class) {
-      /* We haven't seen any packages with this normalized name yet,
-       * so we need to initialize the sequence and add it to the map.
-       */
-      pkg_class = g_sequence_new(NULL);
-      g_sequence_append(pkg_class, pkg_name);
-      g_hash_table_insert( equivalence_classes
-                         , pkg_normalized
-                         , pkg_class
-                         );
-      partition_num = 1;
-    } else {
-      g_free(pkg_normalized);
-      /* Find the package name in the equivalence class */
-      GSequenceIter * pkg_iter =
-        g_sequence_search( pkg_class
-                         , pkg_name
-                         , compare_str
-                         , NULL
-                         );
-      if (!g_sequence_iter_is_end(pkg_iter)) {
-        /* If there are any packages after this one in the list, bail
-         * out. In principle we could solve this by moving them up to
-         * the next partition, but so far I've never seen any github
-         * tarballs out of order so let's save ourselves the work
-         * until we know we need it.
-         */
-        fprintf( stderr
-               , "Out of order github tarball: %s is after %s\n"
-               , pkg_name
-               , (char *) g_sequence_get(pkg_iter)
-               );
-        return 1;
-      }
-      pkg_iter = g_sequence_iter_prev(pkg_iter);
-      if (strcmp( g_sequence_get(pkg_iter)
-                , pkg_name
-                ) != 0) {
-        /* This package doesn't have the same name as the one right
-         * before where it should be in the sequence, which means it's
-         * new and needs to be added to the sequence.
-         *
-         * !!! We need to change this to use g_sequence_insert_before
-         * if we ever get an out-of-order github tarball, see comment
-         * after the check for !g_sequence_iter_is_end(pkg_iter).
-         */
-        pkg_iter = g_sequence_append(pkg_class, pkg_name);
-      } else {
-        g_free(pkg_name);
-      }
-      /* Get the partition number, starting with 1.
-       */
-      partition_num = g_sequence_iter_get_position(pkg_iter) + 1;
-    }
-
-    /* Set the destination path.
-     * The 3 below is for the length of /#/, the partition number part
-     * of the path. If we have more than 9 partitions, we deserve to
-     * segfault. The 1 at the end is for the trailing null.
-     */
-    char * dest_path = g_malloc(output_len + 3 + strlen(path) + 1);
-    sprintf(dest_path, "%s/%d/%s", argv[2], partition_num, path);
-    archive_entry_set_pathname(ent, dest_path);
-
-    if (archive_read_extract(ar, ent, 0) != ARCHIVE_OK) {
-      fprintf( stderr
-             , "Error extracting entry %s from %s: %s\n"
-             , dest_path
-             , argv[1]
-             , archive_error_string(ar)
-             );
-      return 1;
-    }
-  } while ((err = archive_read_next_header(ar, &ent)) == ARCHIVE_OK);
-  if (err != ARCHIVE_EOF) {
-    fprintf( stderr
-           , "Error reading entry from %s: %s\n"
-           , argv[1]
-           , archive_error_string(ar)
-           );
-    return 1;
-  }
-
-  return 0;
-}

pkgs/development/haskell-modules/make-package-set.nix

···
       '';
   };
 
-  all-cabal-hashes-component = name: import (pkgs.runCommand "all-cabal-hashes-component-${name}.nix" {}
-    ''
-      set +o pipefail
-      for component in ${all-cabal-hashes}/*; do
-        if ls $component | grep -q "^${name}$"; then
-          echo "builtins.storePath $component" > $out
-          exit 0
-        fi
-      done
-      echo "${name} not found in any all-cabal-hashes component, are you sure it's in hackage?" >&2
-      exit 1
-    '');
+  all-cabal-hashes-component = name: version: pkgs.runCommand "all-cabal-hashes-component-${name}-${version}" {} ''
+    tar --wildcards -xzvf ${all-cabal-hashes} \*/${name}/${version}/${name}.{json,cabal}
+    mkdir -p $out
+    mv */${name}/${version}/${name}.{json,cabal} $out
+  '';
 
-  hackage2nix = name: version: let component = all-cabal-hashes-component name; in self.haskellSrc2nix {
+  hackage2nix = name: version: let component = all-cabal-hashes-component name version; in self.haskellSrc2nix {
     name   = "${name}-${version}";
-    sha256 = ''$(sed -e 's/.*"SHA256":"//' -e 's/".*$//' "${component}/${name}/${version}/${name}.json")'';
-    src    = "${component}/${name}/${version}/${name}.cabal";
+    sha256 = ''$(sed -e 's/.*"SHA256":"//' -e 's/".*$//' "${component}/${name}.json")'';
+    src    = "${component}/${name}.cabal";
   };
 
 in package-set { inherit pkgs stdenv callPackage; } self // {