release: Revamp release scripts

Covers bins, voices, manuals, fonts, and source tarballs.

Only thing remaining is build-info integration

Change-Id: I3a93fd87bbeb725fc8e1b38c5787e33fa00f3f37

tools/release/README

···
    Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
                      \/            \/     \/    \/            \/
 
- This directory contains useful scripts when creating and building
- releases. The plan is to hopefully create a single top-level script that will
- run everything needed to make a complete and full release and put all
- generated data into a separate output directory.
+ This directory the scripts used to create release artifacts.  All
+ generated data is stored in an output directory (called "output").
+
+ All of these scripts take two arguments; the git tag and the version string.
 
  NOTE that these scripts assume that you have the necessary compilers already
  present in your PATH.
 
  The scripts:
 
- bins.pl          Builds all the binary rockbox zip files.
-
- manuals.pl       Builds all manuals
+   bins.pl          Builds all the binary rockbox zip files
+   manuals.pl       Builds all manuals (pdf and html-zip)
+   sources.sh       Builds the source tarball
+   voices.pl        Builds all "complete enough" voice files
 
- voices.pl        Builds all english voice files
+  Only "stable" targets, their manuals, and a selection of voice files are built;
+  these are designated in tools/builds.pm
 
- tarball.sh       Builds the source tarball
+ Example:
 
+   tools/release/sources.sh v3.15-final 3.15
 
- STATUS
+ Other notes:
 
- All these scripts have a "version" variable at the top that needs editing and
- each of this script needs to be run at the source tree root to function. Most
- of them put their generated files in a subdir called 'output' which the
- scripts create.
+   * Update www/tools/dailybuild-voices.pl file to add the release target
+     voice list!
+   * Update release target in tools/builds.pm

tools/release/bins.pl

···
 #!/usr/bin/perl
-$version="3.15";
-
-require "tools/builds.pm";
+require "./tools/builds.pm";
 
 my $verbose;
 if($ARGV[0] eq "-v") {
···
     shift @ARGV;
 }
 
-my $update;
-if($ARGV[0] eq "-u") {
-    $update =1;
-    shift @ARGV;
-}
+my $tag = $ARGV[0];
+my $version = $ARGV[1];
 
-my $doonly;
-if($ARGV[0]) {
-    $doonly = $ARGV[0];
-    print "only build $doonly\n" if($verbose);
-}
+my $outdir = "output/bins";
 
-if($update) {
-    # svn update!
-    system("svn -q up");
-}
-
-$rev = `svnversion`;
-chomp $rev;
-print "rev $rev\n" if($verbose);
+my $cpus = `nproc`;
 
 # made once for all targets
 sub runone {
     my ($dir, $confnum, $extra)=@_;
     my $a;
 
-    if($doonly && ($doonly ne $dir)) {
-        return;
-    }
-
-    mkdir "build-$dir";
-    chdir "build-$dir";
-    print "Build in build-$dir\n" if($verbose);
+    mkdir "buildb-$dir";
+    chdir "buildb-$dir";
+    print "Build in buildb-$dir\n" if($verbose);
 
     # build the manual(s)
     $a = buildit($dir, $confnum, $extra);
 
     chdir "..";
 
-    my $o="build-$dir/rockbox.zip";
-    my $map="build-$dir/rockbox-maps.zip";
-    my $elf="build-$dir/rockbox-elfs.zip";
+    my $o="buildb-$dir/rockbox.zip";
+    my $map="buildb-$dir/rockbox-maps.zip";
+    my $elf="buildb-$dir/rockbox-elfs.zip";
     if (-f $o) {
-        my $newo="output/rockbox-$dir-$version.zip";
-        my $newmap="output/rockbox-$dir-$version-maps.zip";
-        my $newelf="output/rockbox-$dir-$version-elfs.zip";
-        system("mkdir -p output");
+        my $newo="$outdir/rockbox-$dir-$version.zip";
+        my $newmap="$outdir/rockbox-$dir-$version-maps.zip";
+        my $newelf="$outdir/rockbox-$dir-$version-elfs.zip";
+        system("mkdir -p $outdir");
         system("mv $o $newo");
         print "moved $o to $newo\n" if($verbose);
         system("mv $map $newmap");
···
     my ($dir, $confnum, $newl)=@_;
     my $a;
 
-    if($doonly && ($doonly ne $dir)) {
-        return;
-    }
-
-    mkdir "build-$dir";
-    chdir "build-$dir";
-    print "Build fonts in build-$dir\n" if($verbose);
+    mkdir "buildf-$dir";
+    chdir "buildf-$dir";
+    print "Build fonts in buildf-$dir\n" if($verbose);
 
     # build the fonts
     $a = buildfonts($dir, $confnum, $newl);
 
     chdir "..";
 
-    my $o="build-$dir/rockbox-fonts.zip";
+    my $o="buildf-$dir/rockbox-fonts.zip";
     if (-f $o) {
-        my $newo="output/rockbox-fonts-$version.zip";
+        my $newo="$outdir/rockbox-fonts-$version.zip";
         system("mv $o $newo");
         print "moved $o to $newo\n" if($verbose);
     }
 
-    print "remove all contents in build-$dir\n" if($verbose);
-    system("rm -rf build-$dir");
+    print "remove all contents in buildb-$dir\n" if($verbose);
+    system("rm -rf buildf-$dir");
 
     return $a;
 };
 
-
-
 sub buildit {
     my ($target, $confnum, $extra)=@_;
 
···
     `$c`;
 
     print "Run 'make'\n" if($verbose);
-    `make VERSION=$version`;
+    `make -j$cpus VERSION=$version`;
 
     print "Run 'make zip'\n" if($verbose);
     `make zip VERSION=$version`;
···
     `make fontzip`;
 }
 
+`git checkout $tag`;
+
 # run make in tools first to make sure they're up-to-date
 print "cd tools && make\n" if($verbose);
 `(cd tools && make ) >/dev/null`;
 
-for my $b (&stablebuilds) {
+for my $b (&usablebuilds) {
     my $configname = $builds{$b}{configname} ? $builds{$b}{configname} : $b;
     runone($b, $configname, $builds{$b}{ram});
 }
 
 fonts("fonts", "iaudiox5");
-

tools/release/manuals.pl

···
-#!/usr/bin/perl
+#!/usr/bin/perl -w
+require "./tools/builds.pm";
 
-$version="3.15";
-
-require "tools/builds.pm";
+require "./tools/builds.pm";
 
 my $verbose;
 if($ARGV[0] eq "-v") {
···
     shift @ARGV;
 }
 
-my $doonly;
-if($ARGV[0]) {
-    $doonly = $ARGV[0];
-    print "only build $doonly\n" if($verbose);
-}
+my $tag = $ARGV[0];
+my $version = $ARGV[1];
+
+my $outdir = "output/manuals";
 
 # made once for all targets
 sub runone {
     my ($dir)=@_;
     my $a;
 
-    if($doonly && ($doonly ne $dir)) {
-        return;
-    }
-
     mkdir "buildm-$dir";
     chdir "buildm-$dir";
     print "Build in buildm-$dir\n" if($verbose);
···
 
     chdir "..";
 
-    my $o="buildm-$dir/manual/rockbox-build.pdf";
+    my $o="buildm-$dir/rockbox-$dir-$version.pdf";
     if (-f $o) {
-        my $newo="output/rockbox-$dir-$version.pdf";
+        my $newo="$outdir/rockbox-$dir-$version.pdf";
         system("mv $o $newo");
         print "moved $o to $newo\n" if($verbose);
+    } else {
+	print "buildm-$dir/rockbox-$dir-$version.pdf not found\n";
     }
 
     $o="buildm-$dir/rockbox-manual.zip";
     if (-f $o) {
-        my $newo="output/rockbox-$dir-$version-html.zip";
+        my $newo="$outdir/rockbox-$dir-$version-html.zip";
         system("mv $o $newo");
         print "moved $o to $newo\n" if($verbose);
     }
···
     print "C: $c\n" if($verbose);
     `$c`;
 
-    print "Run 'make'\n" if($verbose);
+    print "Run 'make manual'\n" if($verbose);
     `make manual VERSION=$version 2>/dev/null`;
 
     print "Run 'make manual-zip'\n" if($verbose);
     `make manual-zip VERSION=$version 2>/dev/null`;
 }
+
+`git checkout $tag`;
 
 # run make in tools first to make sure they're up-to-date
 `(cd tools && make ) >/dev/null 2>&1`;
 
-for my $b (&stablebuilds) {
-    next if (length($builds{$b}{configname}) > 0); # no variants
+`mkdir -p $outdir`;
 
+for my $b (&usablebuilds) {
     runone($b);
 }

tools/release/sources.sh

···
+#!/bin/sh
+
+set -e
+set -x
+
+tag=$1
+version=$2
+
+outdir="output/source"
+
+mkdir -p "${outdir}"
+git archive --prefix=rockbox-$version/ -o "${outdir}/rockbox-source-${version}.tar" ${tag}
+xz -f "${outdir}/rockbox-source-${version}.tar"

tools/release/voices.pl

···
-#!/usr/bin/perl
-
-$version="3.15";
-
-require "tools/builds.pm";
+#!/usr/bin/perl -w
+require "./tools/builds.pm";
 
 my $verbose;
 if($ARGV[0] eq "-v") {
···
     shift @ARGV;
 }
 
-my $doonly;
-if($ARGV[0]) {
-    $doonly = $ARGV[0];
-    print "only build $doonly\n" if($verbose);
-}
+my $tag = $ARGV[0];
+my $version = $ARGV[1];
+
+my $outdir = "output/voices";
 
 # made once for all targets
 sub runone {
-    my ($dir)=@_;
+    my ($target, $name, $lang, $engine, $voice, $engine_opts)=@_;
     my $a;
 
-    if($doonly && ($doonly ne $dir)) {
-        return;
-    }
+    print "*** LANGUAGE: $lang\n";
 
-    mkdir "buildv-$dir";
-    chdir "buildv-$dir";
-    print "Build in buildv-$dir\n" if($verbose);
+    print "Build in buildv-$target-$lang\n" if($verbose);
 
-    # build the manual(s)
-    $a = buildit($dir);
+    mkdir "buildv-$target-$lang";
+    chdir "buildv-$target-$lang";
 
-    chdir "..";
+    # build the voice(s)
+    $a = buildit($target, $lang, $engine, $voice, $engine_opts);
 
-    my $o="buildv-$dir/english.voice";
+    my $o="$lang.voice";
     if (-f $o) {
-        my $newo="output/$dir-$version-english.zip";
-        system("cp $o output/$dir-$version-english.voice");
+        my $newo="../$outdir/$target/voice-$target-$version-$name.zip";
+	system("mkdir -p ../$outdir/$target");
+        system("mkdir -p .rockbox/langs");
+        system("mkdir -p output/$target");
         system("mkdir -p .rockbox/langs");
         system("cp $o .rockbox/langs");
-        system("zip -r $newo .rockbox");
+        system("zip -q -r $newo .rockbox");
         system("rm -rf .rockbox");
+        `chmod a+r $newo`;
         print "moved $o to $newo\n" if($verbose);
     }
 
-    print "remove all contents in buildv-$dir\n" if($verbose);
-    system("rm -rf buildv-$dir");
+    chdir "..";
+
+    print "remove all contents in buildv-$target-$lang\n" if($verbose);
+    system("rm -rf buildv-$target-$lang");
 
     return $a;
 };
 
 sub buildit {
-    my ($model)=@_;
+    my ($target, $lang, $engine, $voice, $engine_opts)=@_;
 
     `rm -rf * >/dev/null 2>&1`;
 
-    my $c = "../tools/configure --type=av --target=$model --language=0 --tts=f --ram=0 --voice=0";
+    my $c = "../tools/configure --no-ccache --type=av --target=$target --ram=-1 --language=$lang --tts=$engine --voice=$voice --ttsopts='$engine_opts'";
 
     print "C: $c\n" if($verbose);
-    `$c`;
+    system($c);
 
     print "Run 'make voice'\n" if($verbose);
-    print `make voice VERSION=$version 2>/dev/null`;
+    `make voice`;
 }
+
+`git checkout $tag`;
 
 # run make in tools first to make sure they're up-to-date
 `(cd tools && make ) >/dev/null 2>&1`;
···
 `rm -f $pool/*`;
 $ENV{'POOL'}="$pool";
 
-for my $b (&stablebuilds) {
-    next if (length($builds{$b}{configname}) > 0); # no variants
+`mkdir -p $outdir`;
+
+for my $b (&usablebuilds) {
+    next if ($builds{$b}{voice}); # no variants
+
+    for my $v (&allvoices) {
+	my %voice = $voices{$v};
 
-    runone($b);
-}
+#        print " runone $b $v ($voices{$v}->{lang} via $voices{$v}->{defengine})\n";
+	runone($b, $v, $voices{$v}->{lang}, $voices{$v}->{defengine},
+	       "-1", $voices{$v}->{engines}->{$voices{$v}->{defengine}});
+        }
+    }