#!/usr/bin/env bash # make opencode-e2e driver: the full OpenCode cluster proof against the shared # daevix-dev kind cluster (rootful podman), from repo config only: # # 1. build + load BOTH images (dvx + demo fixture with the probe client) # using the rootful-podman overrides # 2. roll ONLY deployment/dvx-llmproxy and daemonset/dvx-gateway (NEVER the # broker, never any live velverk workload) # 3. author the probe agent (idempotent) + its AGENT-scoped # llm.credential_mode=inject policy (never org scope) # 4. write the org secret llm.api_key through the broker relay (the value # crosses only as command substitution; never printed) # 5. run the probe agent (one turn, then Completed) and print its log # 6. assert the audit evidence in the enclave ClickHouse, scoped by the # probe agent uid AND the run window (never a global count: the tables # carry live velverk traffic and earlier-era rows) # 7. TENANT CASE (bd_2_0-726.5): create a tenant org + foreign (unregistered) # org with their own operators in-script, register tenancy, place a tenant # probe agent with TENANT credentials, author the TENANT org's llm.api_key # via the relay, run the tenant probe, and assert the same evidence keyed # to the tenant coordinate; the unregistered org is refused at placement # (400) and at the broker (403). The enclave is NOT re-registered and the # broker is NEVER rolled; tenancy + seeded rows are cleaned up afterwards. # # Secrets: no secret value is printed, logged, or asserted by its bytes. set -euo pipefail REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) cd "$REPO_ROOT" CLUSTER=${KIND_CLUSTER:-daevix-dev} ORG=${E2E_ORG:-velverk} ENCLAVE_ID=${E2E_ENCLAVE_ID:-} # resolved below (agent create needs the enclave ID, not its name) PROBE_UID=${E2E_PROBE_UID:-opencode-probe} PROBE_IMAGE=${DEMO_IMAGE:-localhost/dvx-demo:dev} ZITADEL_KEY=${DVX_ZITADEL_KEY_FILE:-/home/mrh/.config/velverk/daevix-sa.json} OPENCODE_KEY_FILE=${DVX_E2E_OPENCODE_KEY_FILE:-/home/mrh/opencodego.txt} ISSUER=${DVX_OIDC_ISSUER:-http://zitadel.dvx-system.svc.cluster.local:8080} CP=${DVX_CONTROLPLANE_URL:-https://dvx-controlplane.dvx-system.svc.cluster.local:8443} RELAY=${DVX_RELAY_URL:-https://10.89.0.2:30452} DVX_BIN=${E2E_DVX_BIN:-/tmp/dvx-opencode-e2e} # Rootful podman overrides (this cluster's kind node runs under rootful podman). export PODMAN=${PODMAN:-"sudo -n podman"} export KIND=${KIND:-"sudo -n env KIND_EXPERIMENTAL_PROVIDER=podman kind"} export SUDO=${SUDO:-"sudo -n"} dvxcp() { "$DVX_BIN" "$@" --controlplane "$CP" --insecure-skip-verify --zitadel-key "$ZITADEL_KEY" --issuer "$ISSUER"; } CH() { kubectl exec -i -n dvx-enclave enclave-clickhouse-0 -- clickhouse-client --user dvx --password dvx-dev-enclave-clickhouse --query "$1"; } assert_nonzero() { [ "$1" -ge 1 ] || { echo "FAIL: $2 (got $1)"; exit 1; }; } assert_zero() { [ "$1" -eq 0 ] || { echo "FAIL: $2 (got $1)"; exit 1; }; } fail() { echo "FAIL: $*"; exit 1; } # run_probe_pod UID: the agent pod runs ONE turn and exits, so a Completed pod # from a previous run would not rerun: delete it (the broker reconciles the # desired agent set back), then wait for the pod to appear, run to completion, # and print its log; the log must contain a result=OK leg. run_probe_pod() { local uid=$1 kubectl delete pod -n "$(kubectl get pods -A -l dvx.io/agent-uid=$uid -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null)" \ "$(kubectl get pods -A -l dvx.io/agent-uid=$uid -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" \ --ignore-not-found 2>/dev/null || true local n=0 until kubectl get pods -A -l dvx.io/agent-uid=$uid 2>/dev/null | grep -q Running || [ $n -ge 36 ]; do sleep 5; n=$((n+1)); done local ns pod ns=$(kubectl get pods -A -l dvx.io/agent-uid=$uid -o jsonpath='{.items[0].metadata.namespace}') pod=$(kubectl get pods -A -l dvx.io/agent-uid=$uid -o jsonpath='{.items[0].metadata.name}') [ -n "$ns" ] && [ -n "$pod" ] || fail "probe pod never appeared (broker did not reconcile agent $uid)" n=0 until [ "$(kubectl get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}')" != "Running" ] || [ $n -ge 72 ]; do sleep 5; n=$((n+1)); done kubectl logs -n "$ns" "$pod" --tail=50 || true kubectl logs -n "$ns" "$pod" 2>/dev/null | grep -q 'result=OK' || fail "probe pod log shows no result=OK leg (see logs above)" } command -v go >/dev/null 2>&1 || export PATH="$HOME/.local/share/mise/shims:$PATH" echo ">> [1/6] building + loading images (dvx + demo fixture) with rootful overrides" make kind-load PODMAN="$PODMAN" KIND="$KIND" SUDO="$SUDO" make kind-load-demo PODMAN="$PODMAN" KIND="$KIND" SUDO="$SUDO" echo ">> [2/6] rolling ONLY llmproxy and gateway (never the broker)" export KUBECONFIG=/tmp/daevix.kubeconfig export KIND_EXPERIMENTAL_PROVIDER=podman sudo -n env KIND_EXPERIMENTAL_PROVIDER=podman kind get kubeconfig --name "$CLUSTER" > "$KUBECONFIG" # Re-apply the llmproxy manifest (stamped as deploy-enclave stamps it) so the # agent-identity trust bundle hostPath points at THIS enclave's svids subtree: # an unstamped deployment trusts a stale legacy CA and llmproxy rejects every # force-routed SVID (the same re-stamp deploy-demo performs). PREFIX=$(kubectl get cm dvx-enclave-identity -n dvx-enclave -o jsonpath='{.data.enclave-id}' | cut -c1-12) sed -e "s|path: /run/dvx/svids/ca #|path: /run/dvx/$PREFIX/svids/ca #|" \ deploy/enclave/llmproxy.yaml | kubectl apply -f - kubectl rollout restart deployment/dvx-llmproxy -n dvx-enclave kubectl rollout status deployment/dvx-llmproxy -n dvx-enclave --timeout=150s kubectl rollout restart daemonset/dvx-gateway -n dvx-enclave kubectl rollout status daemonset/dvx-gateway -n dvx-enclave --timeout=120s echo ">> [3/6] building the branch CLI + authoring the probe agent and its agent-scoped inject policy" go build -o "$DVX_BIN" ./cmd/dvx ENCLAVE_ID=${ENCLAVE_ID:-$(kubectl get cm dvx-enclave-identity -n dvx-enclave -o jsonpath='{.data.enclave-id}')} if ! dvxcp agent get --org "$ORG" --uid "$PROBE_UID" >/dev/null 2>&1; then dvxcp agent create --org "$ORG" --uid "$PROBE_UID" --name "$PROBE_UID" --enclave "$ENCLAVE_ID" \ --image "$PROBE_IMAGE" \ --command /bin/sh --command -c \ --command 'DVX_PROBE_BASE_URL=https://opencode.ai /usr/local/bin/demo-opencode-probe' else echo ">> probe agent $PROBE_UID already exists (reusing)" fi # AGENT scope only: org-scope llm.* policy would govern every live velverk agent. dvxcp policy llm credential-mode --mode inject --org "$ORG" --agent "$PROBE_UID" echo ">> [4/6] writing the org secret llm.api_key through the broker relay (value never printed)" if [ -r "$OPENCODE_KEY_FILE" ]; then "$DVX_BIN" secret set llm.api_key --value "$(cat "$OPENCODE_KEY_FILE")" \ --broker-url "$RELAY" --insecure-skip-verify --zitadel-key "$ZITADEL_KEY" --issuer "$ISSUER" else echo ">> key file absent; relying on the already-written org secret llm.api_key" fi # Secret landed, verified WITHOUT exposing the value (51-byte plaintext -> ct_len 228): ct_len=$(kubectl exec -n dvx-enclave deploy/enclave-postgres -- psql -U postgres -d enclave -tAc \ "SELECT length(ciphertext) FROM sovereign_secret WHERE org_id='$ORG' AND name='llm.api_key';") [ "$ct_len" = "228" ] || fail "org secret llm.api_key ct_len=$ct_len, want 228" # Let the policy snapshot + secret settle before the probe runs. sleep 10 echo ">> [5/6] running the probe agent" RUN_START=$(date -u '+%Y-%m-%d %H:%M:%S') RUN_START_RFC=$(date -u '+%Y-%m-%dT%H:%M:%SZ') run_probe_pod "$PROBE_UID" echo ">> [6/6] asserting the audit evidence (scoped by agent uid + run window)" INJ="audit.inference_request WHERE provider='opencode-go' AND agent_id='$PROBE_UID' AND event_time >= '$RUN_START'" chat_rows=$(CH "SELECT count() FROM $INJ AND endpoint='opencode-go:/inference/go/openai/v1/chat/completions' AND credential_mode='inject' AND outcome='allow' AND status_code=200") assert_nonzero "$chat_rows" "no allow/inject/200 inference_request rows for the chat route" zen_rows=$(CH "SELECT count() FROM $INJ AND endpoint='opencode-go:/zen/go/v1/messages' AND credential_mode='inject' AND outcome='allow' AND status_code=200") assert_nonzero "$zen_rows" "no allow/inject/200 inference_request rows for /zen/go/v1/messages" fwd_rows=$(CH "SELECT count() FROM audit.audit_event WHERE action='llmproxy.forward' AND actor='agent:$PROBE_UID' AND attrs['provider']='opencode-go' AND attrs['cred_mode']='inject' AND event_time >= '$RUN_START'") assert_nonzero "$fwd_rows" "no llmproxy.forward events (provider=opencode-go, cred_mode=inject)" flow_rows=$(CH "SELECT count() FROM audit.network_flow WHERE dst_host='opencode.ai' AND actor='agent:$PROBE_UID' AND event_time >= '$RUN_START'") assert_nonzero "$flow_rows" "no gateway network_flow row for dst_host=opencode.ai" models_rows=$(CH "SELECT count() FROM $INJ AND endpoint LIKE '%models'") assert_zero "$models_rows" "the models call WAS force-routed (inference_request row present)" echo ">> inference_request rows for the run:" CH "SELECT event_time, agent_id, provider, endpoint, model, credential_mode, outcome, status_code, record_form FROM $INJ ORDER BY event_time DESC LIMIT 10" echo ">> llmproxy.forward events for the run (header names only; the key value never appears):" CH "SELECT event_time, actor, attrs['provider'] AS provider, attrs['cred_mode'] AS cred_mode, attrs['upstream'] AS upstream, attrs['path'] AS path FROM audit.audit_event WHERE action='llmproxy.forward' AND actor='agent:$PROBE_UID' AND event_time >= '$RUN_START' ORDER BY event_time DESC LIMIT 10" echo ">> credential header discipline (credential_name + header_diff names only):" CH "SELECT credential_name, header_diff FROM $INJ ORDER BY event_time DESC LIMIT 5" echo ">> [7/9] TENANT CASE: provisioning the tenant + unregistered orgs (operators minted in the controlplane pod)" TENANT_ORG=${E2E_TENANT_ORG:-e2etenant} FOREIGN_ORG=${E2E_FOREIGN_ORG:-e2eforeign} TENANT_UID=${E2E_TENANT_UID:-opencode-probe-tenant} KEYDIR=/tmp/dvx-e2e-keys; mkdir -p "$KEYDIR"; chmod 700 "$KEYDIR" TKEY=$KEYDIR/$TENANT_ORG.json; TSUB_FILE=$KEYDIR/$TENANT_ORG.sub FKEY=$KEYDIR/$FOREIGN_ORG.json; FSUB_FILE=$KEYDIR/$FOREIGN_ORG.sub RUN_ID=$(date -u '+%Y%m%d%H%M%S') # key_valid: a key file is reusable only if it still completes the Zitadel # JWT-profile token exchange (orgs/users persist across runs; keys do too, # but /tmp may not). key_valid() { "$DVX_BIN" internal serviceaccount check --key "$1" --issuer "$ISSUER" >/dev/null 2>&1; } # ensure_org NAME: create the controlplane org row + Zitadel org via the # operator API as the velverk service principal (POST /api/v1/orgs is an # upsert, so re-runs are idempotent; the org id IS the chosen name). ensure_org() { "$DVX_BIN" internal serviceaccount check --key "$ZITADEL_KEY" --issuer "$ISSUER" \ --controlplane "$CP" --insecure-skip-verify --operator-create-org "$1" >/dev/null 2>&1 \ || echo ">> org $1 already exists" } # ensure_operator ORG KEYFILE SUBFILE: mint an org-scoped operator machine user # and fetch its key. Operator minting is a controlplane-scoped Zitadel # authority: it MUST run inside the controlplane pod (a host run fails with # Zitadel 404 AUTHZ-cdgFk); the key is copied out via kubectl exec and the # in-pod copy removed. A fresh operator has no dvx grants, so the caller's # first grant is bootstrapped with one direct-DB row (the accepted in-boundary # pattern), cleaned up in phase 9. ensure_operator() { local org=$1 key=$2 subf=$3 if [ -f "$key" ] && key_valid "$key" && [ -f "$subf" ]; then echo ">> operator key for $org reused" return 0 fi ensure_org "$org" kubectl exec -n dvx-system deploy/dvx-controlplane -c controlplane -- \ dvx internal serviceaccount provision-operator --key /zitadel/key.json --issuer "$ISSUER" \ --org "$org" --username "$org-op-$RUN_ID" --out /tmp/op-key.json 2>"$key.provision.log" \ || fail "provision-operator for $org failed (see $key.provision.log)" kubectl exec -n dvx-system deploy/dvx-controlplane -c controlplane -- cat /tmp/op-key.json > "$key" chmod 600 "$key" kubectl exec -n dvx-system deploy/dvx-controlplane -c controlplane -- rm -f /tmp/op-key.json sed -n 's/.*user=\([^ ]*\).*/\1/p' "$key.provision.log" > "$subf" [ -s "$subf" ] || fail "could not read the provisioned operator sub for $org" } # ensure_grant ORG SUB: idempotently seed the org-admin grant row the freshly # provisioned operator needs (no API caller can mint a first grant). ensure_grant() { kubectl exec -n dvx-system deploy/iam-postgres -- psql -U postgres -d controlplane -tAc \ "WITH ins AS (INSERT INTO grant_ (principal, role, scope_kind, org_id) SELECT 'user:$2', 'admin', 'org', '$1' WHERE NOT EXISTS (SELECT 1 FROM grant_ WHERE principal='user:$2' AND scope_kind='org' AND org_id='$1') RETURNING id) SELECT id FROM ins UNION ALL SELECT id FROM grant_ WHERE principal='user:$2' AND scope_kind='org' AND org_id='$1' LIMIT 1" >/dev/null } ensure_operator "$TENANT_ORG" "$TKEY" "$TSUB_FILE" ensure_operator "$FOREIGN_ORG" "$FKEY" "$FSUB_FILE" TSUB=$(cat "$TSUB_FILE"); FSUB=$(cat "$FSUB_FILE") ensure_grant "$TENANT_ORG" "$TSUB" ensure_grant "$FOREIGN_ORG" "$FSUB" dvxcp_t() { "$DVX_BIN" "$@" --controlplane "$CP" --insecure-skip-verify --zitadel-key "$TKEY" --issuer "$ISSUER"; } dvxcp_f() { "$DVX_BIN" "$@" --controlplane "$CP" --insecure-skip-verify --zitadel-key "$FKEY" --issuer "$ISSUER"; } echo ">> [8/9] TENANT CASE: registering tenancy, placing the tenant probe, authoring the tenant secret" # Owner authority: open self-registration, then register the TENANT org (the # seed admin principal gets an enclave-local org-admin grant on snapshot apply). dvxcp enclaves share --enclave-id "$ENCLAVE_ID" dvxcp enclaves tenants add --enclave-id "$ENCLAVE_ID" --org "$TENANT_ORG" --seed-admin "user:$TSUB" # Tenancy snapshot pull + policy/secret settle before the tenant flow. sleep 10 if ! dvxcp_t agent get --org "$TENANT_ORG" --uid "$TENANT_UID" >/dev/null 2>&1; then dvxcp_t agent create --org "$TENANT_ORG" --uid "$TENANT_UID" --name "$TENANT_UID" --enclave "$ENCLAVE_ID" \ --image "$PROBE_IMAGE" \ --command /bin/sh --command -c \ --command 'DVX_PROBE_BASE_URL=https://opencode.ai /usr/local/bin/demo-opencode-probe' else echo ">> tenant probe agent $TENANT_UID already exists (reusing)" fi # AGENT scope only (never org scope). dvxcp_t policy llm credential-mode --mode inject --org "$TENANT_ORG" --agent "$TENANT_UID" # The gateway's destination engine is default-deny per org: allow the probe's # provider host at the tenant AGENT's scope (velverk's own org-scoped allow is # its own policy; the tenant coordinate authoring its own is the point here). dvxcp_t policy destination --org "$TENANT_ORG" --agent "$TENANT_UID" --allow opencode.ai # The TENANT org authors its OWN llm.api_key through the relay (value crosses # only as command substitution; never printed). if [ -r "$OPENCODE_KEY_FILE" ]; then "$DVX_BIN" secret set llm.api_key --value "$(cat "$OPENCODE_KEY_FILE")" \ --broker-url "$RELAY" --insecure-skip-verify --zitadel-key "$TKEY" --issuer "$ISSUER" else echo ">> key file absent; relying on the already-written tenant secret llm.api_key" fi sleep 10 # Negative asserts, SAME run: the unregistered org is refused at agent # placement (400, indistinguishable from unknown enclave) and at the broker's # secret API (403, blanket served-set gate). Status lines land in a temp file, # never secret material. NEGLOG=$KEYDIR/negative.log if dvxcp_f agent create --org "$FOREIGN_ORG" --uid neg-foreign --name neg-foreign --enclave "$ENCLAVE_ID" \ --image "$PROBE_IMAGE" --command /bin/sh --command -c --command 'true' \ >"$NEGLOG" 2>&1; then fail "unregistered org $FOREIGN_ORG was NOT refused at agent placement" fi grep -q "status 400" "$NEGLOG" || fail "unregistered-org placement was not a 400 (see $NEGLOG)" if "$DVX_BIN" secret set neg.probe --value unused --broker-url "$RELAY" --insecure-skip-verify \ --zitadel-key "$FKEY" --issuer "$ISSUER" >>"$NEGLOG" 2>&1; then fail "unregistered org $FOREIGN_ORG was NOT refused at the broker secret API" fi grep -q "status 403" "$NEGLOG" || fail "unregistered-org broker authoring was not a 403 (see $NEGLOG)" rm -f "$NEGLOG" echo ">> [9/9] TENANT CASE: running the tenant probe pod + asserting the tenant coordinate" run_probe_pod "$TENANT_UID" TINJ="audit.inference_request WHERE provider='opencode-go' AND agent_id='$TENANT_UID' AND event_time >= '$RUN_START'" t_chat=$(CH "SELECT count() FROM $TINJ AND endpoint='opencode-go:/inference/go/openai/v1/chat/completions' AND credential_mode='inject' AND outcome='allow' AND status_code=200") assert_nonzero "$t_chat" "no allow/inject/200 inference_request rows for the TENANT chat route" t_zen=$(CH "SELECT count() FROM $TINJ AND endpoint='opencode-go:/zen/go/v1/messages' AND credential_mode='inject' AND outcome='allow' AND status_code=200") assert_nonzero "$t_zen" "no allow/inject/200 inference_request rows for the TENANT /zen route" t_models=$(CH "SELECT count() FROM $TINJ AND endpoint LIKE '%models'") assert_zero "$t_models" "the TENANT models call WAS force-routed (inference_request row present)" t_fwd=$(CH "SELECT count() FROM audit.audit_event WHERE action='llmproxy.forward' AND actor='agent:$TENANT_UID' AND attrs['provider']='opencode-go' AND attrs['cred_mode']='inject' AND event_time >= '$RUN_START'") assert_nonzero "$t_fwd" "no llmproxy.forward events for the TENANT probe" # Audit attribution: the tenant rows carry the TENANT org coordinate. t_org=$(CH "SELECT count() FROM audit.audit_event WHERE org_id='$TENANT_ORG' AND agent_id='$TENANT_UID' AND event_time >= '$RUN_START'") assert_nonzero "$t_org" "tenant audit rows are not attributed to org $TENANT_ORG" # Org-scope discipline: the tenant secret is keyed to the TENANT org (ciphertext # present, length never pinned), the foreign org wrote NOTHING, and velverk's # own secret row is untouched (exactly one). t_ctlen=$(kubectl exec -n dvx-enclave deploy/enclave-postgres -- psql -U postgres -d enclave -tAc \ "SELECT length(ciphertext) FROM sovereign_secret WHERE org_id='$TENANT_ORG' AND name='llm.api_key';") assert_nonzero "${t_ctlen:-0}" "no sovereign_secret row keyed to the TENANT org" f_rows=$(kubectl exec -n dvx-enclave deploy/enclave-postgres -- psql -U postgres -d enclave -tAc \ "SELECT count(*) FROM sovereign_secret WHERE org_id='$FOREIGN_ORG';") assert_zero "${f_rows:-0}" "the unregistered org wrote a secret row despite the 403" v_rows=$(kubectl exec -n dvx-enclave deploy/enclave-postgres -- psql -U postgres -d enclave -tAc \ "SELECT count(*) FROM sovereign_secret WHERE org_id='$ORG' AND name='llm.api_key';") assert_zero $(( v_rows - 1 )) "velverk org's llm.api_key row count changed (want exactly 1)" # Tenant audit persona: the tenant org-admin reads ITS OWN audit through the # relay (broker persona scoping forces the org coordinate server-side). The # id_token is minted per use and captured, never printed. TOK=$("$DVX_BIN" internal serviceaccount check --key "$TKEY" --issuer "$ISSUER" --print-token \ | sed -n 's/^id_token: //p') [ -n "$TOK" ] || fail "could not mint a tenant id_token for the audit persona check" "$DVX_BIN" audit events --source enclave --enclave "$RELAY" --insecure-skip-verify \ --token "$TOK" --agent "$TENANT_UID" --since "$RUN_START_RFC" --limit 50 > "$KEYDIR/tenant-audit.json" grep -q "$TENANT_UID" "$KEYDIR/tenant-audit.json" || fail "tenant audit persona read returned no rows for the tenant agent" rm -f "$KEYDIR/tenant-audit.json" echo ">> [cleanup] removing the tenancy registration + bootstrapped grant rows (shared toggle reverted, tenant agent deleted)" dvxcp_t agent delete --org "$TENANT_ORG" --uid "$TENANT_UID" >/dev/null 2>&1 || true dvxcp enclaves tenants remove --enclave-id "$ENCLAVE_ID" --org "$TENANT_ORG" dvxcp enclaves unshare --enclave-id "$ENCLAVE_ID" kubectl exec -n dvx-system deploy/iam-postgres -- psql -U postgres -d controlplane -tAc \ "DELETE FROM grant_ WHERE principal IN ('user:$TSUB','user:$FSUB');" >/dev/null echo ">> tenant inference_request rows for the run:" CH "SELECT event_time, org_id, agent_id, provider, endpoint, credential_mode, outcome, status_code FROM $TINJ ORDER BY event_time DESC LIMIT 10" echo ">> PASS: make opencode-e2e (velverk probe OK + TENANT probe OK, both inject/200 on both wire families; unregistered org refused 400/403; audit scoped to each probe uid + window; velverk org-scope untouched)"