pyrox.dev
lol
copy-tarballs: Use an S3 bucket for tarballs.nixos.org
Tarballs.nixos.org is now stored in an S3 bucket rather than an EBS
volume. Redirects are used to simulate symlinks.
The function find-tarballs.nix now filters out fetchzip, fetchpatch
and the like.
+119
-67
+112
-65
maintainers/scripts/copy-tarballs.pl
+112
-65
maintainers/scripts/copy-tarballs.pl
···
1
-
#! /run/current-system/sw/bin/perl -w
1
+
#! /usr/bin/env nix-shell
2
+
#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 nixUnstable
3
+
4
+
# This command uploads tarballs to tarballs.nixos.org, the
5
+
# content-addressed cache used by fetchurl as a fallback for when
6
+
# upstream tarballs disappear or change. Usage:
7
+
#
8
+
# 1) To upload a single file:
9
+
#
10
+
# $ copy-tarballs.pl --file /path/to/tarball.tar.gz
11
+
#
12
+
# 2) To upload all files obtained via calls to fetchurl in a Nix derivation:
13
+
#
14
+
# $ copy-tarballs.pl --expr '(import <nixpkgs> {}).hello'
2
15
3
16
use strict;
4
-
use XML::Simple;
17
+
use warnings;
5
18
use File::Basename;
6
19
use File::Path;
7
-
use File::Copy 'cp';
8
-
use IPC::Open2;
20
+
use JSON;
21
+
use Net::Amazon::S3;
9
22
use Nix::Store;
10
23
11
-
my $myDir = dirname($0);
24
+
# S3 setup.
25
+
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die;
26
+
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die;
12
27
13
-
my $tarballsCache = $ENV{'NIX_TARBALLS_CACHE'} // "/tarballs";
28
+
my $s3 = Net::Amazon::S3->new(
29
+
{ aws_access_key_id => $aws_access_key_id,
30
+
aws_secret_access_key => $aws_secret_access_key,
31
+
retry => 1,
32
+
});
14
33
15
-
my $xml = `nix-instantiate --eval-only --xml --strict '<nixpkgs/maintainers/scripts/find-tarballs.nix>'`;
16
-
die "$0: evaluation failed\n" if $? != 0;
34
+
my $bucket = $s3->bucket("nixpkgs-tarballs") or die;
17
35
18
-
my $data = XMLin($xml) or die;
36
+
sub alreadyMirrored {
37
+
my ($algo, $hash) = @_;
38
+
return defined $bucket->get_key("$algo/$hash");
39
+
}
40
+
41
+
sub uploadFile {
42
+
my ($fn, $name) = @_;
19
43
20
-
mkpath($tarballsCache);
21
-
mkpath("$tarballsCache/md5");
22
-
mkpath("$tarballsCache/sha1");
23
-
mkpath("$tarballsCache/sha256");
44
+
my $md5_16 = hashFile("md5", 0, $fn) or die;
45
+
my $sha1_16 = hashFile("sha1", 0, $fn) or die;
46
+
my $sha256_32 = hashFile("sha256", 1, $fn) or die;
47
+
my $sha256_16 = hashFile("sha256", 0, $fn) or die;
48
+
my $sha512_32 = hashFile("sha512", 1, $fn) or die;
49
+
my $sha512_16 = hashFile("sha512", 0, $fn) or die;
24
50
25
-
foreach my $file (@{$data->{list}->{attrs}}) {
26
-
my $url = $file->{attr}->{url}->{string}->{value};
27
-
my $algo = $file->{attr}->{type}->{string}->{value};
28
-
my $hash = $file->{attr}->{hash}->{string}->{value};
51
+
my $mainKey = "sha512/$sha512_16";
29
52
30
-
if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
31
-
print STDERR "skipping $url (unsupported scheme)\n";
32
-
next;
33
-
}
53
+
return if alreadyMirrored("sha512", $sha512_16);
34
54
35
-
$url =~ /([^\/]+)$/;
36
-
my $fn = $1;
55
+
# Upload the file as sha512/<hash-in-base-16>.
56
+
print STDERR "uploading $fn to $mainKey...\n";
57
+
$bucket->add_key_filename($mainKey, $fn, { 'x-amz-meta-original-name' => $name })
58
+
or die "failed to upload $fn to $mainKey\n";
37
59
38
-
if (!defined $fn) {
39
-
print STDERR "skipping $url (no file name)\n";
40
-
next;
60
+
# Create redirects from the other hash types.
61
+
sub redirect {
62
+
my ($name, $dest) = @_;
63
+
#print STDERR "linking $name to $dest...\n";
64
+
$bucket->add_key($name, "", { 'x-amz-website-redirect-location' => "/" . $dest })
65
+
or die "failed to create redirect from $name to $dest\n";
41
66
}
67
+
redirect "md5/$md5_16", $mainKey;
68
+
redirect "sha1/$sha1_16", $mainKey;
69
+
redirect "sha256/$sha256_32", $mainKey;
70
+
redirect "sha256/$sha256_16", $mainKey;
71
+
redirect "sha512/$sha512_32", $mainKey;
72
+
}
42
73
43
-
if ($fn =~ /[&?=%]/ || $fn =~ /^\./) {
44
-
print STDERR "skipping $url (bad character in file name)\n";
45
-
next;
46
-
}
74
+
my $op = $ARGV[0] // "";
47
75
48
-
if ($fn !~ /[a-zA-Z]/) {
49
-
print STDERR "skipping $url (no letter in file name)\n";
50
-
next;
76
+
if ($op eq "--file") {
77
+
my $fn = $ARGV[1] // die "$0: --file requires a file name\n";
78
+
if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) {
79
+
print STDERR "$fn is already mirrored\n";
80
+
} else {
81
+
uploadFile($fn, basename $fn);
51
82
}
83
+
}
52
84
53
-
if ($fn !~ /[0-9]/) {
54
-
print STDERR "skipping $url (no digit in file name)\n";
55
-
next;
56
-
}
85
+
elsif ($op eq "--expr") {
57
86
58
-
if ($fn !~ /[-_\.]/) {
59
-
print STDERR "skipping $url (no dash/dot/underscore in file name)\n";
60
-
next;
61
-
}
87
+
# Evaluate find-tarballs.nix.
88
+
my $expr = $ARGV[1] // die "$0: --expr requires a Nix expression\n";
89
+
my $pid = open(JSON, "-|", "nix-instantiate", "--eval-only", "--json", "--strict",
90
+
"<nixpkgs/maintainers/scripts/find-tarballs.nix>",
91
+
"--arg", "expr", $expr);
92
+
my $stdout = <JSON>;
93
+
waitpid($pid, 0);
94
+
die "$0: evaluation failed\n" if $?;
95
+
close JSON;
62
96
63
-
my $dstPath = "$tarballsCache/$fn";
97
+
my $fetches = decode_json($stdout);
64
98
65
-
next if -e $dstPath;
99
+
print STDERR "evaluation returned ", scalar(@{$fetches}), " tarballs\n";
66
100
67
-
print "downloading $url to $dstPath...\n";
101
+
# Check every fetchurl call discovered by find-tarballs.nix.
102
+
my $mirrored = 0;
103
+
my $have = 0;
104
+
foreach my $fetch (@{$fetches}) {
105
+
my $url = $fetch->{url};
106
+
my $algo = $fetch->{type};
107
+
my $hash = $fetch->{hash};
68
108
69
-
next if $ENV{DRY_RUN};
109
+
if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
110
+
print STDERR "skipping $url (unsupported scheme)\n";
111
+
next;
112
+
}
70
113
71
-
$ENV{QUIET} = 1;
72
-
$ENV{PRINT_PATH} = 1;
73
-
my $fh;
74
-
my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
75
-
waitpid($pid, 0) or die;
76
-
if ($? != 0) {
77
-
print STDERR "failed to fetch $url: $?\n";
78
-
next;
79
-
}
80
-
<$fh>; my $storePath = <$fh>; chomp $storePath;
114
+
if (alreadyMirrored($algo, $hash)) {
115
+
$have++;
116
+
next;
117
+
}
81
118
82
-
die unless -e $storePath;
119
+
print STDERR "mirroring $url...\n";
83
120
84
-
cp($storePath, $dstPath) or die;
121
+
next if $ENV{DRY_RUN};
85
122
86
-
my $md5 = hashFile("md5", 0, $storePath) or die;
87
-
symlink("../$fn", "$tarballsCache/md5/$md5");
123
+
# Download the file using nix-prefetch-url.
124
+
$ENV{QUIET} = 1;
125
+
$ENV{PRINT_PATH} = 1;
126
+
my $fh;
127
+
my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
128
+
waitpid($pid, 0) or die;
129
+
if ($? != 0) {
130
+
print STDERR "failed to fetch $url: $?\n";
131
+
next;
132
+
}
133
+
<$fh>; my $storePath = <$fh>; chomp $storePath;
88
134
89
-
my $sha1 = hashFile("sha1", 0, $storePath) or die;
90
-
symlink("../$fn", "$tarballsCache/sha1/$sha1");
135
+
uploadFile($storePath, $url);
136
+
$mirrored++;
137
+
}
91
138
92
-
my $sha256 = hashFile("sha256", 0, $storePath) or die;
93
-
symlink("../$fn", "$tarballsCache/sha256/$sha256");
139
+
print STDERR "mirrored $mirrored files, already have $have files\n";
140
+
}
94
141
95
-
$sha256 = hashFile("sha256", 1, $storePath) or die;
96
-
symlink("../$fn", "$tarballsCache/sha256/$sha256");
142
+
else {
143
+
die "Syntax: $0 --file FILENAME | --expr EXPR\n";
97
144
}
+7
-2
maintainers/scripts/find-tarballs.nix
+7
-2
maintainers/scripts/find-tarballs.nix
···
4
4
with import ../.. { };
5
5
with lib;
6
6
7
+
{ expr ? removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" ] }:
8
+
7
9
let
8
10
9
-
root = removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" ];
11
+
root = expr;
10
12
11
13
uniqueUrls = map (x: x.file) (genericClosure {
12
14
startSet = map (file: { key = file.url; inherit file; }) urls;
···
15
17
16
18
urls = map (drv: { url = head drv.urls; hash = drv.outputHash; type = drv.outputHashAlgo; }) fetchurlDependencies;
17
19
18
-
fetchurlDependencies = filter (drv: drv.outputHash or "" != "" && drv ? urls) dependencies;
20
+
fetchurlDependencies =
21
+
filter
22
+
(drv: drv.outputHash or "" != "" && drv.outputHashMode == "flat" && drv.postFetch or "" == "" && drv ? urls)
23
+
dependencies;
19
24
20
25
dependencies = map (x: x.value) (genericClosure {
21
26
startSet = map keyDrv (derivationsIn' root);