overby.me
this repo has no description
1#!/usr/bin/env bash
2# run-tests.sh — Compile and run all regression tests inside a Darling prefix
3#
4# This script copies the test source files into the Darling prefix, compiles
5# them using Darling's macOS toolchain (cc), and executes them. It collects
6# results from all test suites and produces a summary.
7#
8# Usage:
9# ./scripts/run-tests.sh [OPTIONS]
10#
11# Options:
12# --prefix <path> Darling prefix path (default: ~/.darling or $DPREFIX)
13# --suite <name> Run only the named suite (can be repeated)
14# Available: renameatx_np, setattrlist_flags, utimensat,
15# sandbox_api, sandbox_exec, dirserv
16# --keep Keep compiled test binaries in the prefix after running
17# --verbose Show full test output even on success
18# --help Show this help message
19#
20# Prerequisites:
21# - Darling must be installed and `darling shell echo ok` must work
22# - The Darling prefix must be initialized
23#
24# Exit code:
25# 0 — all tests passed
26# 1 — one or more tests failed
27# 2 — infrastructure error (Darling not working, compilation failure, etc.)
28#
29# See: plan/PLAN.md "What's Next" item 1
30
31set -euo pipefail
32
33SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
34REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
35
36# ── Defaults ────────────────────────────────────────────────────────────────
37
38DARLING_PREFIX="${DPREFIX:-$HOME/.darling}"
39KEEP_BINARIES=0
40VERBOSE=0
41SELECTED_SUITES=()
42
43# Test directory inside the Darling prefix
44DARLING_TEST_DIR="/tmp/darling-nix-tests"
45
46# ── Colors ──────────────────────────────────────────────────────────────────
47
48if [ -t 1 ]; then
49 RED='\033[0;31m'
50 GREEN='\033[0;32m'
51 YELLOW='\033[0;33m'
52 BLUE='\033[0;34m'
53 BOLD='\033[1m'
54 DIM='\033[2m'
55 RESET='\033[0m'
56else
57 RED='' GREEN='' YELLOW='' BLUE='' BOLD='' DIM='' RESET=''
58fi
59
60# ── Helpers ─────────────────────────────────────────────────────────────────
61
62log() { echo -e "${GREEN}[run-tests]${RESET} $*"; }
63warn() { echo -e "${YELLOW}[run-tests] WARNING:${RESET} $*" >&2; }
64err() { echo -e "${RED}[run-tests] ERROR:${RESET} $*" >&2; }
65fatal() { err "$@"; exit 2; }
66
67dsh() {
68 darling shell "$@"
69}
70
71usage() {
72 cat <<'EOF'
73Usage: ./scripts/run-tests.sh [OPTIONS]
74
75Compile and run all Darling regression tests.
76
77Options:
78 --prefix <path> Darling prefix (default: ~/.darling or $DPREFIX)
79 --suite <name> Run only the named suite (repeatable)
80 Available: renameatx_np, setattrlist_flags, utimensat,
81 sandbox_api, sandbox_exec, dirserv
82 --keep Keep compiled binaries in the prefix after running
83 --verbose Show full test output even on success
84 --help Show this help
85
86Suites:
87 renameatx_np — renameatx_np syscall 488 (5 tests)
88 setattrlist_flags — setattrlist/getattrlist ATTR_CMN_FLAGS (10 tests)
89 utimensat — utimensat/setattrlistat timestamps (16 tests)
90 sandbox_api — sandbox C API stubs
91 sandbox_exec — sandbox-exec integration (shell tests)
92 dirserv — Directory Services stubs (dseditgroup, sysadminctl, dscl)
93EOF
94 exit 0
95}
96
97# ── Argument Parsing ────────────────────────────────────────────────────────
98
99while [ $# -gt 0 ]; do
100 case "$1" in
101 --prefix)
102 [ $# -ge 2 ] || fatal "--prefix requires an argument"
103 DARLING_PREFIX="$2"
104 shift 2
105 ;;
106 --suite)
107 [ $# -ge 2 ] || fatal "--suite requires an argument"
108 SELECTED_SUITES+=("$2")
109 shift 2
110 ;;
111 --keep)
112 KEEP_BINARIES=1
113 shift
114 ;;
115 --verbose|-v)
116 VERBOSE=1
117 shift
118 ;;
119 --help|-h)
120 usage
121 ;;
122 *)
123 fatal "Unknown option: $1 (try --help)"
124 ;;
125 esac
126done
127
128# ── Suite Definitions ───────────────────────────────────────────────────────
129
130# Each suite is defined as:
131# SUITE_<name>_TYPE = "c" | "sh"
132# SUITE_<name>_SOURCE = relative path from repo root
133# SUITE_<name>_DESC = human description
134# SUITE_<name>_CFLAGS = extra compiler flags (C suites only)
135
136declare -A SUITE_TYPE SUITE_SOURCE SUITE_DESC SUITE_CFLAGS
137
138SUITE_TYPE[renameatx_np]="c"
139SUITE_SOURCE[renameatx_np]="tests/syscall/test_renameatx_np.c"
140SUITE_DESC[renameatx_np]="renameatx_np (syscall 488) — plain rename, SWAP, EXCL, invalid flags"
141SUITE_CFLAGS[renameatx_np]=""
142
143SUITE_TYPE[setattrlist_flags]="c"
144SUITE_SOURCE[setattrlist_flags]="tests/syscall/test_setattrlist_flags.c"
145SUITE_DESC[setattrlist_flags]="setattrlist/getattrlist ATTR_CMN_FLAGS — lchflags, chflags, combined attrs"
146SUITE_CFLAGS[setattrlist_flags]=""
147
148SUITE_TYPE[utimensat]="c"
149SUITE_SOURCE[utimensat]="tests/syscall/test_utimensat.c"
150SUITE_DESC[utimensat]="utimensat/setattrlistat — timestamps, MODTIME, ACCTIME, CRTIME, symlinks"
151SUITE_CFLAGS[utimensat]=""
152
153SUITE_TYPE[sandbox_api]="c"
154SUITE_SOURCE[sandbox_api]="tests/sandbox/test_sandbox_api.c"
155SUITE_DESC[sandbox_api]="sandbox C API — sandbox_init, sandbox_free_error"
156SUITE_CFLAGS[sandbox_api]=""
157
158SUITE_TYPE[sandbox_exec]="sh"
159SUITE_SOURCE[sandbox_exec]="tests/sandbox/test_sandbox_exec.sh"
160SUITE_DESC[sandbox_exec]="sandbox-exec stub — flag parsing, exec, exit codes, Nix patterns"
161SUITE_CFLAGS[sandbox_exec]=""
162
163SUITE_TYPE[dirserv]="sh"
164SUITE_SOURCE[dirserv]="tests/dirserv/test_dirserv.sh"
165SUITE_DESC[dirserv]="Directory Services stubs — dseditgroup, sysadminctl, dscl (Phase 5.1)"
166SUITE_CFLAGS[dirserv]=""
167
168ALL_SUITES=(renameatx_np setattrlist_flags utimensat sandbox_api sandbox_exec dirserv)
169
170# ── Determine which suites to run ──────────────────────────────────────────
171
172if [ ${#SELECTED_SUITES[@]} -eq 0 ]; then
173 RUN_SUITES=("${ALL_SUITES[@]}")
174else
175 RUN_SUITES=()
176 for s in "${SELECTED_SUITES[@]}"; do
177 if [ -z "${SUITE_TYPE[$s]+x}" ]; then
178 fatal "Unknown suite: $s (available: ${ALL_SUITES[*]})"
179 fi
180 RUN_SUITES+=("$s")
181 done
182fi
183
184# ── Preflight ───────────────────────────────────────────────────────────────
185
186log "${BOLD}Preflight checks...${RESET}"
187
188if ! command -v darling &>/dev/null; then
189 fatal "darling is not installed or not in PATH"
190fi
191
192if [ ! -d "$DARLING_PREFIX" ]; then
193 fatal "Darling prefix not found at $DARLING_PREFIX\n" \
194 " Initialize with: darling shell true"
195fi
196
197if ! dsh echo ok &>/dev/null; then
198 fatal "darling shell is not functional\n" \
199 " Try: darling shell echo ok"
200fi
201
202log " Prefix: $DARLING_PREFIX"
203log " Suites: ${RUN_SUITES[*]}"
204
205# Verify source files exist
206for suite in "${RUN_SUITES[@]}"; do
207 src="$REPO_DIR/${SUITE_SOURCE[$suite]}"
208 if [ ! -f "$src" ]; then
209 fatal "Source file not found: ${SUITE_SOURCE[$suite]}\n" \
210 " Expected at: $src"
211 fi
212done
213
214# ── Copy test sources into the prefix ───────────────────────────────────────
215
216log "${BOLD}Copying test sources into Darling prefix...${RESET}"
217
218PREFIX_TEST_DIR="$DARLING_PREFIX/private/tmp/darling-nix-tests"
219mkdir -p "$PREFIX_TEST_DIR"
220
221for suite in "${RUN_SUITES[@]}"; do
222 src="$REPO_DIR/${SUITE_SOURCE[$suite]}"
223 cp "$src" "$PREFIX_TEST_DIR/"
224 log " Copied ${SUITE_SOURCE[$suite]}"
225done
226
227# ── Compile C test suites ──────────────────────────────────────────────────
228
229log "${BOLD}Compiling C test suites inside Darling...${RESET}"
230
231COMPILE_FAILURES=0
232
233for suite in "${RUN_SUITES[@]}"; do
234 if [ "${SUITE_TYPE[$suite]}" != "c" ]; then
235 continue
236 fi
237
238 src_basename="$(basename "${SUITE_SOURCE[$suite]}")"
239 bin_name="${src_basename%.c}"
240
241 log " Compiling $src_basename ..."
242
243 compile_output=$(dsh bash -c \
244 "cd $DARLING_TEST_DIR && cc -Wall -Wextra -o '$bin_name' '$src_basename' ${SUITE_CFLAGS[$suite]} 2>&1" \
245 2>&1) || {
246 err " Compilation of $src_basename FAILED:"
247 echo "$compile_output" | sed 's/^/ /' >&2
248 COMPILE_FAILURES=$((COMPILE_FAILURES + 1))
249 continue
250 }
251
252 if [ "$VERBOSE" -eq 1 ] && [ -n "$compile_output" ]; then
253 echo "$compile_output" | sed 's/^/ /'
254 fi
255
256 log " ${GREEN}✓${RESET} $bin_name compiled"
257done
258
259if [ "$COMPILE_FAILURES" -gt 0 ]; then
260 err "$COMPILE_FAILURES suite(s) failed to compile"
261 if [ "$COMPILE_FAILURES" -eq "${#RUN_SUITES[@]}" ]; then
262 fatal "All suites failed to compile — is the Darling toolchain working?"
263 fi
264fi
265
266# ── Run test suites ─────────────────────────────────────────────────────────
267
268echo ""
269log "${BOLD}═══════════════════════════════════════════════════════════${RESET}"
270log "${BOLD} Running Darling regression tests${RESET}"
271log "${BOLD}═══════════════════════════════════════════════════════════${RESET}"
272echo ""
273
274SUITES_PASSED=0
275SUITES_FAILED=0
276SUITES_SKIPPED=0
277
278declare -A SUITE_RESULTS # "pass", "fail", "skip"
279declare -A SUITE_OUTPUT
280
281for suite in "${RUN_SUITES[@]}"; do
282 src_basename="$(basename "${SUITE_SOURCE[$suite]}")"
283 echo -e "${BLUE}━━━ Suite: $suite ━━━${RESET}"
284 echo -e "${DIM}${SUITE_DESC[$suite]}${RESET}"
285 echo ""
286
287 if [ "${SUITE_TYPE[$suite]}" = "c" ]; then
288 bin_name="${src_basename%.c}"
289
290 # Check the binary was compiled
291 compile_check=$(dsh test -x "$DARLING_TEST_DIR/$bin_name" 2>&1 && echo "yes" || echo "no")
292 if [ "$compile_check" != "yes" ]; then
293 echo -e " ${YELLOW}SKIPPED${RESET} — compilation failed"
294 echo ""
295 SUITES_SKIPPED=$((SUITES_SKIPPED + 1))
296 SUITE_RESULTS[$suite]="skip"
297 continue
298 fi
299
300 # Run the test binary
301 output=""
302 exit_code=0
303 output=$(dsh bash -c "cd $DARLING_TEST_DIR && ./$bin_name 2>&1" 2>&1) || exit_code=$?
304
305 elif [ "${SUITE_TYPE[$suite]}" = "sh" ]; then
306 # Shell tests run directly
307 output=""
308 exit_code=0
309 output=$(dsh bash -c "cd $DARLING_TEST_DIR && sh '$src_basename' 2>&1" 2>&1) || exit_code=$?
310 fi
311
312 SUITE_OUTPUT[$suite]="$output"
313
314 if [ "$exit_code" -eq 0 ]; then
315 SUITES_PASSED=$((SUITES_PASSED + 1))
316 SUITE_RESULTS[$suite]="pass"
317 if [ "$VERBOSE" -eq 1 ]; then
318 echo "$output"
319 echo ""
320 fi
321 echo -e " ${GREEN}✓ PASSED${RESET}"
322 else
323 SUITES_FAILED=$((SUITES_FAILED + 1))
324 SUITE_RESULTS[$suite]="fail"
325 # Always show output on failure
326 echo "$output"
327 echo ""
328 echo -e " ${RED}✗ FAILED${RESET} (exit code: $exit_code)"
329 fi
330
331 echo ""
332done
333
334# ── Cleanup ─────────────────────────────────────────────────────────────────
335
336if [ "$KEEP_BINARIES" -eq 0 ]; then
337 log "Cleaning up test files from prefix..."
338 rm -rf "$PREFIX_TEST_DIR"
339else
340 log "Test binaries preserved at: $DARLING_TEST_DIR (inside prefix)"
341 log " Host path: $PREFIX_TEST_DIR"
342fi
343
344# ── Summary ─────────────────────────────────────────────────────────────────
345
346TOTAL=$((SUITES_PASSED + SUITES_FAILED + SUITES_SKIPPED))
347
348echo ""
349log "${BOLD}═══════════════════════════════════════════════════════════${RESET}"
350log "${BOLD} Test Summary${RESET}"
351log "${BOLD}═══════════════════════════════════════════════════════════${RESET}"
352echo ""
353
354# Per-suite results table
355printf " %-25s %s\n" "Suite" "Result"
356printf " %-25s %s\n" "─────────────────────────" "──────"
357for suite in "${RUN_SUITES[@]}"; do
358 result="${SUITE_RESULTS[$suite]:-skip}"
359 case "$result" in
360 pass) printf " %-25s ${GREEN}✓ PASS${RESET}\n" "$suite" ;;
361 fail) printf " %-25s ${RED}✗ FAIL${RESET}\n" "$suite" ;;
362 skip) printf " %-25s ${YELLOW}⊘ SKIP${RESET}\n" "$suite" ;;
363 esac
364done
365
366echo ""
367printf " Total: %d suites\n" "$TOTAL"
368printf " Passed: ${GREEN}%d${RESET}\n" "$SUITES_PASSED"
369printf " Failed: ${RED}%d${RESET}\n" "$SUITES_FAILED"
370printf " Skipped: ${YELLOW}%d${RESET}\n" "$SUITES_SKIPPED"
371echo ""
372
373if [ "$SUITES_FAILED" -gt 0 ]; then
374 err "Some test suites failed."
375 echo ""
376 echo "Debugging tips:" >&2
377 echo " • Re-run with --verbose to see full output for passing tests" >&2
378 echo " • Run a single suite: $0 --suite <name>" >&2
379 echo " • Run with --keep to preserve binaries, then inspect inside:" >&2
380 echo " darling shell $DARLING_TEST_DIR/<test_binary>" >&2
381 echo " • Trace syscalls: strace -f -p \$(pidof darlingserver) 2>&1 | head" >&2
382 echo " • Darling xtrace: DARLING_XTRACE=1 darling shell $DARLING_TEST_DIR/<test_binary>" >&2
383 echo ""
384 exit 1
385elif [ "$SUITES_SKIPPED" -gt 0 ] && [ "$SUITES_PASSED" -eq 0 ]; then
386 warn "All suites were skipped — check compilation errors above."
387 exit 2
388else
389 log "${GREEN}All tests passed!${RESET}"
390 exit 0
391fi