Something went wrong. Try again.
The fastest decode and prefill for 4-bit Qwen3.8-27B on one RTX 3090 that preserves the model's intelligence
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261#!/usr/bin/env bash# Invoke native upstream evaluation. No prompts, task selection, or scoring here.set -euo pipefailset +xexport LC_ALL=Cunset PYTHONPATH PYTHONHOMEumask 077root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)repo=$(cd -- "$root/.." && pwd -P)usage() { printf '%s\n' 'Usage: eval/scripts/run PROFILE [TASKSET] [--dry-run] [--measure-power] [--output FRESH_DIRECTORY]' \ ' tiny aime25 Three native seed-0 shuffled math tasks; not full qualification.' \ ' broad [TASKSET] Broad autoresearch suite tasksets after tiny/aime25: mmlu-pro 10, i3-logic 4, livecodebench 3.' \ ' diverse i3-logic First native eligible logic task; separate 8192-token sentinel.' \ ' --output FRESH_DIRECTORY Exclusive-create the result group at this path.' \ ' --measure-power Collect board energy and per-environment task efficiency.' \ ' --power-gpu=0 One GPU index, UUID or PCI bus ID (no power-policy changes).' \ ' --power-interval-seconds=0.5 Monotonic sample period; allowed 0.1..60 s.' \ ' --power-max-seconds=86400 Stop collection, not evaluation, after 1..604800 s.' \ ' --power-max-samples=200000 Bound retained observations to 2..1000000.' \ 'QWEN_SERVING_CONTAINER selects the running EXL3 serving container (default: the one running container publishing the configs/local.toml endpoint).' \ 'Measurement is opt-in, cannot accompany --dry-run, and never changes upstream scores.'}if [[ ${1:-} == --help || ${1:-} == -h ]]; then usage; exit 0; fiprofile=${1:?Usage: eval/scripts/run PROFILE [TASKSET] [--dry-run] [--measure-power] [--output FRESH_DIRECTORY]}shiftcase "$profile" in smoke|tiny|broad|diverse|quick|full|agentic|agentic-full) ;; *) printf 'Unknown profile: %s\n' "$profile" >&2; exit 2 ;; esacselection=""dry=falsemeasure=falseoutput=""power_options=()while [[ $# -gt 0 ]]; do argument=$1 shift if [[ "$argument" == --dry-run ]]; then dry=true elif [[ "$argument" == --measure-power ]]; then measure=true elif [[ "$argument" == --output ]]; then [[ -z "$output" && $# -gt 0 && -n "$1" ]] || { usage >&2; exit 2; } output=$1 shift elif [[ "$argument" == --output=* ]]; then [[ -z "$output" && -n "${argument#*=}" ]] || { usage >&2; exit 2; } output=${argument#*=} elif [[ "$argument" == --power-gpu=* ]]; then power_options+=("--gpu=${argument#*=}") elif [[ "$argument" == --power-interval-seconds=* ]]; then power_options+=("--interval-seconds=${argument#*=}") elif [[ "$argument" == --power-max-seconds=* ]]; then power_options+=("--max-seconds=${argument#*=}") elif [[ "$argument" == --power-max-samples=* ]]; then power_options+=("--max-samples=${argument#*=}") elif [[ -z "$selection" && "$argument" =~ ^[a-z0-9-]+$ ]]; then selection=$argument else usage >&2; exit 2 fidoneif [[ "$measure" == false && ${#power_options[@]} -gt 0 ]]; then printf 'Power options require --measure-power\n' >&2; exit 2fiif [[ "$measure" == true && "$dry" == true ]]; then printf 'Power measurement requires a real evaluation, not --dry-run\n' >&2; exit 2fiif [[ -n "$output" && ( -e "$output" || -L "$output" ) ]]; then printf 'Output already exists; choose a fresh directory: %s\n' "$output" >&2; exit 2fiif [[ "$profile" == tiny && -n "$selection" && "$selection" != aime25 ]]; then printf 'Tiny is sampled math only: aime25, 3 tasks, 1 rollout each\n' >&2; exit 2fiif [[ "$profile" == diverse && -n "$selection" && "$selection" != i3-logic ]]; then printf 'Diverse is the separate logic sentinel only: i3-logic, 1 task, 1 rollout\n' >&2; exit 2fiif [[ -n "$selection" ]]; then configs=("$root/configs/$profile/$selection.toml") [[ -f "${configs[0]}" ]] || { printf 'No such profile taskset: %s\n' "$selection" >&2; exit 2; }else configs=("$root/configs/$profile/"*.toml)fi[[ -x "$root/.venv/bin/eval" ]] || { printf 'Run eval/scripts/setup first\n' >&2; exit 1; }"$root/scripts/setup" --checkif [[ "$dry" == false ]]; then : "${QWEN_API_KEY_FILE:?Set the absolute path to the serving API key file}" [[ "$QWEN_API_KEY_FILE" == /* && -f "$QWEN_API_KEY_FILE" && -r "$QWEN_API_KEY_FILE" ]] || { printf 'API key file must be absolute and readable\n' >&2; exit 1; } [[ $(stat --format='%u' "$QWEN_API_KEY_FILE") -eq $(id -u) ]] case $(stat --format='%a' "$QWEN_API_KEY_FILE") in 400|600) ;; *) printf 'API key must be private (0400/0600)\n' >&2; exit 1 ;; esac [[ $(wc -c <"$QWEN_API_KEY_FILE") -le 4097 ]] grep -Iq . "$QWEN_API_KEY_FILE" || { printf 'API key must be nonempty text without NUL bytes\n' >&2; exit 1; } QWEN_API_KEY=$(cat -- "$QWEN_API_KEY_FILE") [[ "$QWEN_API_KEY" =~ ^[!-~]+$ && ${#QWEN_API_KEY} -le 4096 ]] || { printf 'Invalid API key file\n' >&2; exit 1; } export QWEN_API_KEY for config in "${configs[@]}"; do taskset=$(basename -- "$config" .toml) case "$taskset" in mrcr-v2-*) taskset=mrcr-v2 ;; esac "$root/scripts/data" --check "$taskset" done # Actual serving container: the named or the one running container that # publishes the configured endpoint. Fail before any output or request. endpoint=$(sed -n 's|^base_url = "http://\([0-9.]*:[0-9]*\)/v1"$|\1|p' "$root/configs/local.toml") [[ "$endpoint" =~ ^[0-9.]+:[0-9]+$ ]] || { printf 'Unsupported configs/local.toml base_url\n' >&2; exit 1; } published() { docker container inspect --format \ '{{range $port, $bindings := .NetworkSettings.Ports}}{{range $bindings}}{{.HostIp}}:{{.HostPort}} {{end}}{{end}}' \ -- "$1" } if [[ -n "${QWEN_SERVING_CONTAINER:-}" ]]; then serving=$QWEN_SERVING_CONTAINER bindings=$(published "$serving") [[ " $bindings " == *" $endpoint "* ]] || { printf 'Serving container %s does not publish %s\n' "$serving" "$endpoint" >&2; exit 1; } else containers=$(docker container ls --no-trunc --quiet) matches=() for container in $containers; do bindings=$(published "$container") if [[ " $bindings " == *" $endpoint "* ]]; then matches+=("$container"); fi done [[ ${#matches[@]} -eq 1 ]] || { printf 'Expected one running container publishing %s; set QWEN_SERVING_CONTAINER\n' "$endpoint" >&2; exit 1; } serving=${matches[0]} fifi# Upstream subprocess runtime has a HOME-relative cache. Never use the user's# home for runtime installers, scripts, platform login, or result state.export HOME="$root/.cache/home"export HF_HOME="$root/.cache/huggingface"export HF_HUB_CACHE="$HF_HOME/hub"export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 HF_HUB_DISABLE_TELEMETRY=1export MRCR_V2_CACHE_DIR="$root/.cache/mrcr_v2"export UV_PYTHON="$root/.venv/bin/python" UV_PYTHON_DOWNLOADS=neverunset UV_FROZENmkdir -p -- "$HOME" "$root/runs"if [[ -n "$output" ]]; then mkdir -- "$output" group=$(realpath -- "$output")else group=$(mktemp -d "$root/runs/$profile.XXXXXXXX")fi# Fresh derived Arrow cache: only authenticated raw dataset assets are reused.export HF_DATASETS_CACHE="$group/.dataset-cache"printf 'Native outputs and provenance: %s\n' "$group"mkdir -- "$group/provenance"cp -- "$root/prime-envs.lock" "$root/datasets.lock" "$root/uv.lock" "$root/configs/local.toml" "$group/provenance/"cp -R -- "$root/runtime" "$group/provenance/runtime"cp -- "$root/pyproject.toml" "$repo/flake.lock" "$group/provenance/"sha256sum -- "$root/scripts/"* >"$group/provenance/scripts.sha256"if [[ "$measure" == true ]]; then cp -- "$root/measure.py" "$repo/bench/power.py" "$repo/bench/process.py" "$group/provenance/"figit -C "$repo" rev-parse HEAD >"$group/provenance/recipe-checkout.txt"git -C "$repo" status --porcelain=v1 >"$group/provenance/worktree-status.txt"git -C "$repo" diff --binary HEAD | sha256sum >"$group/provenance/worktree-diff.sha256"cp -- "$repo/prepare/exl3-manifest.json" "$group/provenance/model-artifact-manifest.json"uv --version >"$group/provenance/uv-version.txt""$root/.venv/bin/python" --version >"$group/provenance/python-version.txt"# Actual serving container/image identity; a restart or replacement mid-run fails.serving_identity() { docker container inspect --format \ '{{.Id}} {{.Image}} {{.State.Running}} {{.State.StartedAt}} {{.RestartCount}}' \ -- "$serving"}# Frozen evaluator inputs, rehashed after the run.input_hashes() { sha256sum -- "$1" "$root/configs/local.toml" "$root/datasets.lock" \ "$root/prime-envs.lock" "$root/uv.lock" "$root/pyproject.toml"}cd -- "$root/.sources/prime-envs"for config in "${configs[@]}"; do cd -- "$root/.sources/prime-envs" taskset=$(basename -- "$config" .toml) launch="$group/provenance/$taskset.toml" cp -- "$config" "$launch" if grep -q '^type = "docker"$' "$config" && [[ "$dry" == false ]]; then "$root/scripts/sandbox" --check [[ -f "$root/.cache/sandbox-image" ]] || { printf 'Run eval/scripts/sandbox first\n' >&2; exit 1; } image=$(cat -- "$root/.cache/sandbox-image") [[ "$image" =~ ^sha256:[a-f0-9]{64}$ ]] || { printf 'Invalid sandbox image identity\n' >&2; exit 1; } [[ $(docker image inspect --format '{{.Id}}' "$image") == "$image" ]] sed "s|^image = .*|image = \"$image\"|" "$config" >"$launch" fi flags=(--no-rich --no-push) case "$taskset" in aime24|aime25|aime26|i3-logic) dataset_repo=$(jq -r --arg key "$taskset" '.huggingface[$key].repo' "$root/datasets.lock") revision=$(jq -r --arg key "$taskset" '.huggingface[$key].revision' "$root/datasets.lock") snapshot="$HF_HUB_CACHE/datasets--${dataset_repo//\//--}/snapshots/$revision" # Native dataset-name config accepts the unchanged local snapshot. # This avoids online Hub metadata lookup with a fresh Arrow cache. if [[ "$taskset" == i3-logic ]]; then flags+=(--env.taskset.dataset.name "$snapshot/logic" --env.taskset.dataset.subset default) else flags+=(--env.taskset.dataset-name "$snapshot") fi ;; esac if [[ "$taskset" == graphwalks || "$taskset" == mmlu-pro ]]; then dataset_repo=$(jq -r --arg key "$taskset" '.huggingface[$key].repo' "$root/datasets.lock") revision=$(jq -r --arg key "$taskset" '.huggingface[$key].revision' "$root/datasets.lock") snapshot="$HF_HUB_CACHE/datasets--${dataset_repo//\//--}/snapshots/$revision" # GraphWalks and MMLU-Pro hardcode their Hub names (MMLU-Pro also its # revision, equal to the lock) and take no dataset-name option. # datasets.load_dataset natively resolves this relative local directory # before trying the Hub. mkdir -p -- "$group/local-datasets/${dataset_repo%%/*}" ln -s -- "$snapshot" "$group/local-datasets/$dataset_repo" cd -- "$group/local-datasets" fi pwd -P >"$group/provenance/$taskset-working-directory.txt" if [[ "$profile" == smoke ]]; then flags+=(-v); fi if [[ "$dry" == true ]]; then flags+=(--dry-run); fi native=(uv run --project "$root" --no-sync eval @ "$root/configs/local.toml" @ "$launch" -o "$group/$taskset" "${flags[@]}") before="$group/provenance/$taskset.serving-before.txt" frozen="$group/provenance/$taskset.evaluation-inputs.sha256" if [[ "$dry" == false ]]; then input_hashes "$config" >"$frozen" # Fail before any native evaluator request on a missing or stopped container. serving_identity >"$before" read -r _ _ running _ <"$before" [[ "$running" == true ]] || { printf 'Serving container is not running\n' >&2; exit 1; } fi status=0 if [[ "$measure" == true ]]; then native_cwd=$PWD ( cd -- "$repo" exec "$root/.venv/bin/python" -m eval.measure \ --environment "$taskset" --output-dir "$group/$taskset" \ --working-directory "$native_cwd" "${power_options[@]}" -- "${native[@]}" ) & measurement_pid=$! # Forward wrapper-only signals too; wait for the sidecar to stop sampling # and persist evidence before this shell exits. trap 'kill -INT "$measurement_pid" 2>/dev/null || printf "Measurement already exited\n" >&2' INT trap 'kill -TERM "$measurement_pid" 2>/dev/null || printf "Measurement already exited\n" >&2' TERM trap 'kill -HUP "$measurement_pid" 2>/dev/null || printf "Measurement already exited\n" >&2' HUP status=0 while true; do if wait "$measurement_pid"; then status=0; else status=$?; fi if ! kill -0 "$measurement_pid" 2>/dev/null; then break; fi done trap - INT TERM HUP else if "${native[@]}"; then status=0; else status=$?; fi fi if [[ "$dry" == false ]]; then # Preserve post-run evidence even when the native evaluator exits nonzero. serving_identity >"$group/provenance/$taskset.serving-after.txt" cmp -s -- "$before" "$group/provenance/$taskset.serving-after.txt" || { printf 'Serving container changed during evaluation\n' >&2; exit 1; } input_hashes "$config" | cmp -s -- "$frozen" - || { printf 'Evaluator inputs changed during evaluation\n' >&2; exit 1; } if [[ "$status" -ne 0 ]]; then exit "$status"; fi traces=("$group/$taskset/"*/traces.jsonl) [[ ${#traces[@]} -eq 1 && -s "${traces[0]}" ]] || { printf 'No unique upstream traces file\n' >&2; exit 1; } # Operational failures are not a score. Preserve the native records and # stop rather than let a successful CLI exit hide failed episodes. if jq -se 'length > 0 and all(.ok == true)' "${traces[0]}" >/dev/null; then printf 'Native episodes completed: %s\n' "${traces[0]}" else printf 'Upstream episode failures; inspect %s\n' "${traces[0]}" >&2 jq -r 'select(.ok == false) | .errors[]? | .message' "${traces[0]}" >&2 exit 1 fi fi if [[ "$status" -ne 0 ]]; then exit "$status"; fidone