tjh.dev
1#!/usr/bin/env Rscript
2library(data.table)
3library(parallel)
4library(BiocManager)
5cl <- makeCluster(10)
6
7biocVersion <- BiocManager:::.version_map()
8biocVersion <- biocVersion[biocVersion$R == getRversion()[, 1:2],c("Bioc", "BiocStatus")]
9if ("release" %in% biocVersion$BiocStatus) {
10 biocVersion <- as.numeric(as.character(biocVersion[biocVersion$BiocStatus == "release", "Bioc"]))
11} else {
12 biocVersion <- max(as.numeric(as.character(biocVersion$Bioc)))
13}
14
15mirrorUrls <- list( bioc=paste0("http://bioconductor.statistik.tu-dortmund.de/packages/", biocVersion, "/bioc/src/contrib/")
16 , "bioc-annotation"=paste0("http://bioconductor.statistik.tu-dortmund.de/packages/", biocVersion, "/data/annotation/src/contrib/")
17 , "bioc-experiment"=paste0("http://bioconductor.statistik.tu-dortmund.de/packages/", biocVersion, "/data/experiment/src/contrib/")
18 , cran="https://cran.r-project.org/src/contrib/"
19 )
20
21mirrorType <- commandArgs(trailingOnly=TRUE)[1]
22stopifnot(mirrorType %in% names(mirrorUrls))
23packagesFile <- paste(mirrorType, 'packages.nix', sep='-')
24readFormatted <- as.data.table(read.table(skip=8, sep='"', text=head(readLines(packagesFile), -1)))
25
26write(paste("downloading package lists"), stderr())
27knownPackages <- lapply(mirrorUrls, function(url) as.data.table(available.packages(url, filters=c("R_version", "OS_type", "duplicates")), method="libcurl"))
28pkgs <- knownPackages[mirrorType][[1]]
29setkey(pkgs, Package)
30knownPackages <- c(unique(do.call("rbind", knownPackages)$Package))
31knownPackages <- sapply(knownPackages, gsub, pattern=".", replacement="_", fixed=TRUE)
32
33mirrorUrl <- mirrorUrls[mirrorType][[1]]
34nixPrefetch <- function(name, version) {
35 prevV <- readFormatted$V2 == name & readFormatted$V4 == version
36 if (sum(prevV) == 1)
37 as.character(readFormatted$V6[ prevV ])
38
39 else {
40 # avoid nix-prefetch-url because it often fails to fetch/hash large files
41 url <- paste0(mirrorUrl, name, "_", version, ".tar.gz")
42 tmp <- tempfile(pattern=paste0(name, "_", version), fileext=".tar.gz")
43 cmd <- paste0("wget -q -O '", tmp, "' '", url, "'")
44 if(mirrorType == "cran"){
45 archiveUrl <- paste0(mirrorUrl, "Archive/", name, "/", name, "_", version, ".tar.gz")
46 cmd <- paste0(cmd, " || wget -q -O '", tmp, "' '", archiveUrl, "'")
47 }
48 cmd <- paste0(cmd, " && nix-hash --type sha256 --base32 --flat '", tmp, "'")
49 cmd <- paste0(cmd, " && echo >&2 ' added ", name, " v", version, "'")
50 cmd <- paste0(cmd, " ; rm -rf '", tmp, "'")
51 system(cmd, intern=TRUE)
52 }
53
54}
55
56escapeName <- function(name) {
57 switch(name, "import" = "r_import", "assert" = "r_assert", name)
58}
59
60formatPackage <- function(name, version, sha256, depends, imports, linkingTo) {
61 attr <- gsub(".", "_", escapeName(name), fixed=TRUE)
62 options(warn=5)
63 depends <- paste( if (is.na(depends)) "" else gsub("[ \t\n]+", "", depends)
64 , if (is.na(imports)) "" else gsub("[ \t\n]+", "", imports)
65 , if (is.na(linkingTo)) "" else gsub("[ \t\n]+", "", linkingTo)
66 , sep=","
67 )
68 depends <- unlist(strsplit(depends, split=",", fixed=TRUE))
69 depends <- lapply(depends, gsub, pattern="([^ \t\n(]+).*", replacement="\\1")
70 depends <- lapply(depends, gsub, pattern=".", replacement="_", fixed=TRUE)
71 depends <- depends[depends %in% knownPackages]
72 depends <- lapply(depends, escapeName)
73 depends <- paste(depends)
74 depends <- paste(sort(unique(depends)), collapse=" ")
75 paste0(" ", attr, " = derive2 { name=\"", name, "\"; version=\"", version, "\"; sha256=\"", sha256, "\"; depends=[", depends, "]; };")
76}
77
78clusterExport(cl, c("nixPrefetch","readFormatted", "mirrorUrl", "mirrorType", "knownPackages"))
79
80pkgs <- pkgs[order(Package)]
81
82write(paste("updating", mirrorType, "packages"), stderr())
83pkgs$sha256 <- parApply(cl, pkgs, 1, function(p) nixPrefetch(p[1], p[2]))
84nix <- apply(pkgs, 1, function(p) formatPackage(p[1], p[2], p[18], p[4], p[5], p[6]))
85write("done", stderr())
86
87# Mark deleted packages as broken
88setkey(readFormatted, V2)
89markBroken <- function(name) {
90 str <- paste0(readFormatted[name], collapse='"')
91 if(sum(grep("broken = true;", str)))
92 return(str)
93 write(paste("marked", name, "as broken"), stderr())
94 gsub("};$", "broken = true; };", str)
95}
96broken <- lapply(setdiff(readFormatted[[2]], pkgs[[1]]), markBroken)
97
98cat("# This file is generated from generate-r-packages.R. DO NOT EDIT.\n")
99cat("# Execute the following command to update the file.\n")
100cat("#\n")
101cat(paste("# Rscript generate-r-packages.R", mirrorType, ">new && mv new", packagesFile))
102cat("\n\n")
103cat("{ self, derive }:\n")
104cat("let derive2 = derive ")
105if (mirrorType == "cran") { cat("{ }")
106} else if (mirrorType == "irkernel") { cat("{}")
107} else { cat("{ biocVersion = \"", biocVersion, "\"; }", sep="") }
108cat(";\n")
109cat("in with self; {\n")
110cat(paste(nix, collapse="\n"), "\n", sep="")
111cat(paste(broken, collapse="\n"), "\n", sep="")
112cat("}\n")
113
114stopCluster(cl)