lol

copy-tarballs: Use an S3 bucket for tarballs.nixos.org

Tarballs.nixos.org is now stored in an S3 bucket rather than an EBS
volume. Redirects are used to simulate symlinks.

The function find-tarballs.nix now filters out fetchzip, fetchpatch
and the like.

+119 -67
+112 -65
maintainers/scripts/copy-tarballs.pl
··· 1 - #! /run/current-system/sw/bin/perl -w 1 + #! /usr/bin/env nix-shell 2 + #! nix-shell -i perl -p perl perlPackages.NetAmazonS3 nixUnstable 3 + 4 + # This command uploads tarballs to tarballs.nixos.org, the 5 + # content-addressed cache used by fetchurl as a fallback for when 6 + # upstream tarballs disappear or change. Usage: 7 + # 8 + # 1) To upload a single file: 9 + # 10 + # $ copy-tarballs.pl --file /path/to/tarball.tar.gz 11 + # 12 + # 2) To upload all files obtained via calls to fetchurl in a Nix derivation: 13 + # 14 + # $ copy-tarballs.pl --expr '(import <nixpkgs> {}).hello' 2 15 3 16 use strict; 4 - use XML::Simple; 17 + use warnings; 5 18 use File::Basename; 6 19 use File::Path; 7 - use File::Copy 'cp'; 8 - use IPC::Open2; 20 + use JSON; 21 + use Net::Amazon::S3; 9 22 use Nix::Store; 10 23 11 - my $myDir = dirname($0); 24 + # S3 setup. 25 + my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die; 26 + my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die; 12 27 13 - my $tarballsCache = $ENV{'NIX_TARBALLS_CACHE'} // "/tarballs"; 28 + my $s3 = Net::Amazon::S3->new( 29 + { aws_access_key_id => $aws_access_key_id, 30 + aws_secret_access_key => $aws_secret_access_key, 31 + retry => 1, 32 + }); 14 33 15 - my $xml = `nix-instantiate --eval-only --xml --strict '<nixpkgs/maintainers/scripts/find-tarballs.nix>'`; 16 - die "$0: evaluation failed\n" if $? != 0; 34 + my $bucket = $s3->bucket("nixpkgs-tarballs") or die; 17 35 18 - my $data = XMLin($xml) or die; 36 + sub alreadyMirrored { 37 + my ($algo, $hash) = @_; 38 + return defined $bucket->get_key("$algo/$hash"); 39 + } 40 + 41 + sub uploadFile { 42 + my ($fn, $name) = @_; 19 43 20 - mkpath($tarballsCache); 21 - mkpath("$tarballsCache/md5"); 22 - mkpath("$tarballsCache/sha1"); 23 - mkpath("$tarballsCache/sha256"); 44 + my $md5_16 = hashFile("md5", 0, $fn) or die; 45 + my $sha1_16 = hashFile("sha1", 0, $fn) or die; 46 + my $sha256_32 = hashFile("sha256", 1, $fn) or die; 47 + my $sha256_16 = hashFile("sha256", 0, $fn) or die; 48 + my $sha512_32 = hashFile("sha512", 1, $fn) or die; 49 + my $sha512_16 = hashFile("sha512", 0, $fn) or die; 24 50 25 - foreach my $file (@{$data->{list}->{attrs}}) { 26 - my $url = $file->{attr}->{url}->{string}->{value}; 27 - my $algo = $file->{attr}->{type}->{string}->{value}; 28 - my $hash = $file->{attr}->{hash}->{string}->{value}; 51 + my $mainKey = "sha512/$sha512_16"; 29 52 30 - if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) { 31 - print STDERR "skipping $url (unsupported scheme)\n"; 32 - next; 33 - } 53 + return if alreadyMirrored("sha512", $sha512_16); 34 54 35 - $url =~ /([^\/]+)$/; 36 - my $fn = $1; 55 + # Upload the file as sha512/<hash-in-base-16>. 56 + print STDERR "uploading $fn to $mainKey...\n"; 57 + $bucket->add_key_filename($mainKey, $fn, { 'x-amz-meta-original-name' => $name }) 58 + or die "failed to upload $fn to $mainKey\n"; 37 59 38 - if (!defined $fn) { 39 - print STDERR "skipping $url (no file name)\n"; 40 - next; 60 + # Create redirects from the other hash types. 61 + sub redirect { 62 + my ($name, $dest) = @_; 63 + #print STDERR "linking $name to $dest...\n"; 64 + $bucket->add_key($name, "", { 'x-amz-website-redirect-location' => "/" . $dest }) 65 + or die "failed to create redirect from $name to $dest\n"; 41 66 } 67 + redirect "md5/$md5_16", $mainKey; 68 + redirect "sha1/$sha1_16", $mainKey; 69 + redirect "sha256/$sha256_32", $mainKey; 70 + redirect "sha256/$sha256_16", $mainKey; 71 + redirect "sha512/$sha512_32", $mainKey; 72 + } 42 73 43 - if ($fn =~ /[&?=%]/ || $fn =~ /^\./) { 44 - print STDERR "skipping $url (bad character in file name)\n"; 45 - next; 46 - } 74 + my $op = $ARGV[0] // ""; 47 75 48 - if ($fn !~ /[a-zA-Z]/) { 49 - print STDERR "skipping $url (no letter in file name)\n"; 50 - next; 76 + if ($op eq "--file") { 77 + my $fn = $ARGV[1] // die "$0: --file requires a file name\n"; 78 + if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) { 79 + print STDERR "$fn is already mirrored\n"; 80 + } else { 81 + uploadFile($fn, basename $fn); 51 82 } 83 + } 52 84 53 - if ($fn !~ /[0-9]/) { 54 - print STDERR "skipping $url (no digit in file name)\n"; 55 - next; 56 - } 85 + elsif ($op eq "--expr") { 57 86 58 - if ($fn !~ /[-_\.]/) { 59 - print STDERR "skipping $url (no dash/dot/underscore in file name)\n"; 60 - next; 61 - } 87 + # Evaluate find-tarballs.nix. 88 + my $expr = $ARGV[1] // die "$0: --expr requires a Nix expression\n"; 89 + my $pid = open(JSON, "-|", "nix-instantiate", "--eval-only", "--json", "--strict", 90 + "<nixpkgs/maintainers/scripts/find-tarballs.nix>", 91 + "--arg", "expr", $expr); 92 + my $stdout = <JSON>; 93 + waitpid($pid, 0); 94 + die "$0: evaluation failed\n" if $?; 95 + close JSON; 62 96 63 - my $dstPath = "$tarballsCache/$fn"; 97 + my $fetches = decode_json($stdout); 64 98 65 - next if -e $dstPath; 99 + print STDERR "evaluation returned ", scalar(@{$fetches}), " tarballs\n"; 66 100 67 - print "downloading $url to $dstPath...\n"; 101 + # Check every fetchurl call discovered by find-tarballs.nix. 102 + my $mirrored = 0; 103 + my $have = 0; 104 + foreach my $fetch (@{$fetches}) { 105 + my $url = $fetch->{url}; 106 + my $algo = $fetch->{type}; 107 + my $hash = $fetch->{hash}; 68 108 69 - next if $ENV{DRY_RUN}; 109 + if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) { 110 + print STDERR "skipping $url (unsupported scheme)\n"; 111 + next; 112 + } 70 113 71 - $ENV{QUIET} = 1; 72 - $ENV{PRINT_PATH} = 1; 73 - my $fh; 74 - my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die; 75 - waitpid($pid, 0) or die; 76 - if ($? != 0) { 77 - print STDERR "failed to fetch $url: $?\n"; 78 - next; 79 - } 80 - <$fh>; my $storePath = <$fh>; chomp $storePath; 114 + if (alreadyMirrored($algo, $hash)) { 115 + $have++; 116 + next; 117 + } 81 118 82 - die unless -e $storePath; 119 + print STDERR "mirroring $url...\n"; 83 120 84 - cp($storePath, $dstPath) or die; 121 + next if $ENV{DRY_RUN}; 85 122 86 - my $md5 = hashFile("md5", 0, $storePath) or die; 87 - symlink("../$fn", "$tarballsCache/md5/$md5"); 123 + # Download the file using nix-prefetch-url. 124 + $ENV{QUIET} = 1; 125 + $ENV{PRINT_PATH} = 1; 126 + my $fh; 127 + my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die; 128 + waitpid($pid, 0) or die; 129 + if ($? != 0) { 130 + print STDERR "failed to fetch $url: $?\n"; 131 + next; 132 + } 133 + <$fh>; my $storePath = <$fh>; chomp $storePath; 88 134 89 - my $sha1 = hashFile("sha1", 0, $storePath) or die; 90 - symlink("../$fn", "$tarballsCache/sha1/$sha1"); 135 + uploadFile($storePath, $url); 136 + $mirrored++; 137 + } 91 138 92 - my $sha256 = hashFile("sha256", 0, $storePath) or die; 93 - symlink("../$fn", "$tarballsCache/sha256/$sha256"); 139 + print STDERR "mirrored $mirrored files, already have $have files\n"; 140 + } 94 141 95 - $sha256 = hashFile("sha256", 1, $storePath) or die; 96 - symlink("../$fn", "$tarballsCache/sha256/$sha256"); 142 + else { 143 + die "Syntax: $0 --file FILENAME | --expr EXPR\n"; 97 144 }
+7 -2
maintainers/scripts/find-tarballs.nix
··· 4 4 with import ../.. { }; 5 5 with lib; 6 6 7 + { expr ? removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" ] }: 8 + 7 9 let 8 10 9 - root = removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" ]; 11 + root = expr; 10 12 11 13 uniqueUrls = map (x: x.file) (genericClosure { 12 14 startSet = map (file: { key = file.url; inherit file; }) urls; ··· 15 17 16 18 urls = map (drv: { url = head drv.urls; hash = drv.outputHash; type = drv.outputHashAlgo; }) fetchurlDependencies; 17 19 18 - fetchurlDependencies = filter (drv: drv.outputHash or "" != "" && drv ? urls) dependencies; 20 + fetchurlDependencies = 21 + filter 22 + (drv: drv.outputHash or "" != "" && drv.outputHashMode == "flat" && drv.postFetch or "" == "" && drv ? urls) 23 + dependencies; 19 24 20 25 dependencies = map (x: x.value) (genericClosure { 21 26 startSet = map keyDrv (derivationsIn' root);