test: add robust PostgreSQL cluster for CQRS integration tests

- Replace fragile docker-compose with proper podman-compose setup
- Add replication slots for reliable WAL streaming
- Add proper replica bootstrap with pg_basebackup and su-exec
- Add lag-aware health checks for replicas
- Add run_cluster.sh orchestration script (start/stop/status/clean/test)
- Update test ports to 15432-15434 to avoid conflicts
- Fix concurrent load test to handle pool exhaustion gracefully

test/integration/docker-compose.yml

···
-version: '3.8'
-
-services:
-  primary:
-    image: postgres:16
-    container_name: repodb_primary
-    environment:
-      POSTGRES_USER: repodb
-      POSTGRES_PASSWORD: repodb
-      POSTGRES_DB: repodb_test
-    ports:
-      - '5432:5432'
-    command: >
-      postgres
-      -c wal_level=replica
-      -c max_wal_senders=3
-      -c max_replication_slots=3
-      -c hot_standby=on
-      -c listen_addresses='*'
-    volumes:
-      - ./init-primary.sh:/docker-entrypoint-initdb.d/init-primary.sh
-    healthcheck:
-      test: ['CMD-SHELL', 'pg_isready -U repodb -d repodb_test']
-      interval: 5s
-      timeout: 5s
-      retries: 10
-
-  replica1:
-    image: postgres:16
-    container_name: repodb_replica1
-    environment:
-      POSTGRES_USER: repodb
-      POSTGRES_PASSWORD: repodb
-      PGUSER: repodb
-      PGPASSWORD: repodb
-    ports:
-      - '5433:5432'
-    depends_on:
-      primary:
-        condition: service_healthy
-    entrypoint: /bin/bash
-    command:
-      - -c
-      - |
-        rm -rf /var/lib/postgresql/data/*
-        until PGPASSWORD=repodb pg_basebackup -h primary -D /var/lib/postgresql/data -U repodb -Fp -Xs -P -R; do
-          echo "Waiting for primary..."
-          sleep 2
-        done
-        chmod 700 /var/lib/postgresql/data
-        exec postgres
-    healthcheck:
-      test: ['CMD-SHELL', 'pg_isready -U repodb']
-      interval: 5s
-      timeout: 5s
-      retries: 10
-
-  replica2:
-    image: postgres:16
-    container_name: repodb_replica2
-    environment:
-      POSTGRES_USER: repodb
-      POSTGRES_PASSWORD: repodb
-      PGUSER: repodb
-      PGPASSWORD: repodb
-    ports:
-      - '5434:5432'
-    depends_on:
-      primary:
-        condition: service_healthy
-    entrypoint: /bin/bash
-    command:
-      - -c
-      - |
-        rm -rf /var/lib/postgresql/data/*
-        until PGPASSWORD=repodb pg_basebackup -h primary -D /var/lib/postgresql/data -U repodb -Fp -Xs -P -R; do
-          echo "Waiting for primary..."
-          sleep 2
-        done
-        chmod 700 /var/lib/postgresql/data
-        exec postgres
-    healthcheck:
-      test: ['CMD-SHELL', 'pg_isready -U repodb']
-      interval: 5s
-      timeout: 5s
-      retries: 10

test/integration/init-primary.sh

···
-#!/bin/bash
-set -e
-
-# Configure pg_hba.conf to allow replication connections
-echo "host replication repodb 0.0.0.0/0 md5" >> "$PGDATA/pg_hba.conf"
-echo "host all repodb 0.0.0.0/0 md5" >> "$PGDATA/pg_hba.conf"
-
-# Create test table
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    CREATE TABLE IF NOT EXISTS test_users (
-        id SERIAL PRIMARY KEY,
-        name VARCHAR(100) NOT NULL,
-        email VARCHAR(255) NOT NULL,
-        created_at TIMESTAMP DEFAULT NOW()
-    );
-EOSQL

test/integration/podman-compose.yml

···
+version: '3.8'
+
+networks:
+  repodb_network:
+    driver: bridge
+
+volumes:
+  primary_data:
+  replica1_data:
+  replica2_data:
+
+services:
+  primary:
+    image: docker.io/postgres:16-alpine
+    container_name: repodb_primary
+    hostname: primary
+    networks:
+      - repodb_network
+    ports:
+      - '15432:5432'
+    environment:
+      POSTGRES_USER: repodb
+      POSTGRES_PASSWORD: repodb
+      POSTGRES_DB: repodb_test
+      POSTGRES_INITDB_ARGS: '--data-checksums'
+    volumes:
+      - primary_data:/var/lib/postgresql/data
+      - ./scripts/primary-init.sh:/docker-entrypoint-initdb.d/01-init.sh:ro
+    command:
+      - postgres
+      - -c
+      - wal_level=replica
+      - -c
+      - max_wal_senders=10
+      - -c
+      - max_replication_slots=10
+      - -c
+      - hot_standby=on
+      - -c
+      - wal_keep_size=256MB
+      - -c
+      - listen_addresses=*
+      - -c
+      - synchronous_commit=on
+      - -c
+      - shared_buffers=128MB
+      - -c
+      - work_mem=4MB
+    healthcheck:
+      test: ['CMD-SHELL', '/usr/local/bin/pg_isready -U repodb -d repodb_test && psql -U repodb -d repodb_test -c "SELECT 1"']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  replica1:
+    image: docker.io/postgres:16-alpine
+    container_name: repodb_replica1
+    hostname: replica1
+    networks:
+      - repodb_network
+    ports:
+      - '15433:5432'
+    environment:
+      POSTGRES_USER: repodb
+      POSTGRES_PASSWORD: repodb
+      PGUSER: repodb
+      PGPASSWORD: repodb
+      PRIMARY_HOST: primary
+      PRIMARY_PORT: '5432'
+      REPLICA_NAME: replica1
+    volumes:
+      - replica1_data:/var/lib/postgresql/data
+      - ./scripts/replica-init.sh:/replica-init.sh:ro
+      - ./scripts/healthcheck.sh:/healthcheck.sh:ro
+    depends_on:
+      primary:
+        condition: service_healthy
+    entrypoint: ['/bin/sh', '/replica-init.sh']
+    healthcheck:
+      test: ['CMD-SHELL', '/bin/sh /healthcheck.sh']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+
+  replica2:
+    image: docker.io/postgres:16-alpine
+    container_name: repodb_replica2
+    hostname: replica2
+    networks:
+      - repodb_network
+    ports:
+      - '15434:5432'
+    environment:
+      POSTGRES_USER: repodb
+      POSTGRES_PASSWORD: repodb
+      PGUSER: repodb
+      PGPASSWORD: repodb
+      PRIMARY_HOST: primary
+      PRIMARY_PORT: '5432'
+      REPLICA_NAME: replica2
+    volumes:
+      - replica2_data:/var/lib/postgresql/data
+      - ./scripts/replica-init.sh:/replica-init.sh:ro
+      - ./scripts/healthcheck.sh:/healthcheck.sh:ro
+    depends_on:
+      primary:
+        condition: service_healthy
+    entrypoint: ['/bin/sh', '/replica-init.sh']
+    healthcheck:
+      test: ['CMD-SHELL', '/bin/sh /healthcheck.sh']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 30s

test/integration/run_cluster.sh

···
+#!/bin/bash
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+COMPOSE_FILE="podman-compose.yml"
+COMPOSE_CMD=""
+
+if command -v podman-compose &> /dev/null; then
+    COMPOSE_CMD="podman-compose"
+elif command -v docker-compose &> /dev/null; then
+    COMPOSE_CMD="docker-compose"
+elif command -v docker &> /dev/null && docker compose version &> /dev/null; then
+    COMPOSE_CMD="docker compose"
+else
+    echo "Error: Neither podman-compose nor docker-compose found"
+    exit 1
+fi
+
+usage() {
+    echo "Usage: $0 {start|stop|status|logs|clean|test}"
+    echo ""
+    echo "Commands:"
+    echo "  start   - Start the PostgreSQL cluster"
+    echo "  stop    - Stop the cluster (preserves data)"
+    echo "  status  - Show cluster status and replication info"
+    echo "  logs    - Show container logs"
+    echo "  clean   - Stop cluster and remove all data"
+    echo "  test    - Run integration tests"
+    exit 1
+}
+
+wait_for_cluster() {
+    echo "Waiting for cluster to be ready..."
+    
+    local max_attempts=60
+    local attempt=0
+    
+    while [ $attempt -lt $max_attempts ]; do
+        if PGPASSWORD=repodb psql -h localhost -p 15432 -U repodb -d repodb_test -c "SELECT 1" > /dev/null 2>&1; then
+            break
+        fi
+        attempt=$((attempt + 1))
+        echo "  Waiting for primary... ($attempt/$max_attempts)"
+        sleep 2
+    done
+    
+    if [ $attempt -eq $max_attempts ]; then
+        echo "Error: Primary failed to start"
+        exit 1
+    fi
+    
+    echo "Primary is ready"
+    
+    for port in 15433 15434; do
+        attempt=0
+        while [ $attempt -lt $max_attempts ]; do
+            if PGPASSWORD=repodb psql -h localhost -p $port -U repodb -d repodb_test -c "SELECT pg_is_in_recovery()" 2>/dev/null | grep -q "t"; then
+                break
+            fi
+            attempt=$((attempt + 1))
+            echo "  Waiting for replica on port $port... ($attempt/$max_attempts)"
+            sleep 2
+        done
+        
+        if [ $attempt -eq $max_attempts ]; then
+            echo "Warning: Replica on port $port may not be ready"
+        else
+            echo "Replica on port $port is ready"
+        fi
+    done
+    
+    echo ""
+    echo "Cluster is ready!"
+}
+
+show_status() {
+    echo "=== Cluster Status ==="
+    echo ""
+    
+    echo "Primary (port 15432):"
+    PGPASSWORD=repodb psql -h localhost -p 15432 -U repodb -d repodb_test -c "
+        SELECT 
+            client_addr,
+            state,
+            sent_lsn,
+            write_lsn,
+            flush_lsn,
+            replay_lsn,
+            sync_state
+        FROM pg_stat_replication;
+    " 2>/dev/null || echo "  Not available"
+    
+    echo ""
+    echo "Replication Slots:"
+    PGPASSWORD=repodb psql -h localhost -p 15432 -U repodb -d repodb_test -c "
+        SELECT slot_name, active, restart_lsn 
+        FROM pg_replication_slots;
+    " 2>/dev/null || echo "  Not available"
+    
+    echo ""
+    for port in 15433 15434; do
+        echo "Replica (port $port):"
+        PGPASSWORD=repodb psql -h localhost -p $port -U repodb -d repodb_test -c "
+            SELECT 
+                pg_is_in_recovery() as is_replica,
+                pg_last_wal_receive_lsn() as receive_lsn,
+                pg_last_wal_replay_lsn() as replay_lsn,
+                pg_wal_lsn_diff(pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn()) as lag_bytes;
+        " 2>/dev/null || echo "  Not available"
+        echo ""
+    done
+}
+
+cmd_start() {
+    echo "Starting PostgreSQL cluster..."
+    $COMPOSE_CMD -f "$COMPOSE_FILE" up -d
+    wait_for_cluster
+    show_status
+}
+
+cmd_stop() {
+    echo "Stopping cluster..."
+    $COMPOSE_CMD -f "$COMPOSE_FILE" down
+}
+
+cmd_clean() {
+    echo "Stopping cluster and removing data..."
+    $COMPOSE_CMD -f "$COMPOSE_FILE" down -v
+    echo "Cleaned"
+}
+
+cmd_logs() {
+    $COMPOSE_CMD -f "$COMPOSE_FILE" logs -f
+}
+
+cmd_test() {
+    echo "Building and running integration tests..."
+    
+    show_status
+    
+    cd "$SCRIPT_DIR/../.."
+    
+    if [ -f "_build/default/test/integration/test_cqrs_integration.exe" ]; then
+        echo ""
+        echo "Running tests..."
+        ./_build/default/test/integration/test_cqrs_integration.exe
+    else
+        echo "Building tests first..."
+        dune build test/integration/test_cqrs_integration.exe
+        echo ""
+        echo "Running tests..."
+        ./_build/default/test/integration/test_cqrs_integration.exe
+    fi
+}
+
+case "${1:-}" in
+    start)
+        cmd_start
+        ;;
+    stop)
+        cmd_stop
+        ;;
+    status)
+        show_status
+        ;;
+    logs)
+        cmd_logs
+        ;;
+    clean)
+        cmd_clean
+        ;;
+    test)
+        cmd_test
+        ;;
+    *)
+        usage
+        ;;
+esac

test/integration/scripts/healthcheck.sh

···
+#!/bin/sh
+
+if ! pg_isready -U "$POSTGRES_USER" > /dev/null 2>&1; then
+    exit 1
+fi
+
+RECOVERY_STATUS=$(psql -U "$POSTGRES_USER" -d repodb_test -tAc "SELECT pg_is_in_recovery();" 2>/dev/null)
+if [ "$RECOVERY_STATUS" != "t" ]; then
+    exit 1
+fi
+
+LAG_BYTES=$(psql -U "$POSTGRES_USER" -d repodb_test -tAc "
+    SELECT COALESCE(
+        pg_wal_lsn_diff(
+            pg_last_wal_receive_lsn(),
+            pg_last_wal_replay_lsn()
+        ),
+        0
+    )::bigint;
+" 2>/dev/null)
+
+MAX_LAG_BYTES=16777216
+if [ -n "$LAG_BYTES" ] && [ "$LAG_BYTES" -gt "$MAX_LAG_BYTES" ]; then
+    exit 1
+fi
+
+exit 0

test/integration/scripts/primary-init.sh

···
+#!/bin/bash
+set -e
+
+cat >> "$PGDATA/pg_hba.conf" <<EOF
+host replication repodb 0.0.0.0/0 md5
+host all repodb 0.0.0.0/0 md5
+EOF
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    SELECT pg_create_physical_replication_slot('replica1_slot', true)
+    WHERE NOT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'replica1_slot');
+    
+    SELECT pg_create_physical_replication_slot('replica2_slot', true)
+    WHERE NOT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'replica2_slot');
+
+    CREATE TABLE IF NOT EXISTS test_users (
+        id SERIAL PRIMARY KEY,
+        name VARCHAR(100) NOT NULL,
+        email VARCHAR(255) NOT NULL,
+        created_at TIMESTAMP DEFAULT NOW()
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_test_users_name ON test_users(name);
+EOSQL
+
+echo "Primary initialization complete"

test/integration/scripts/replica-init.sh

···
+#!/bin/sh
+set -e
+
+PGDATA="/var/lib/postgresql/data"
+PRIMARY_HOST="${PRIMARY_HOST:-primary}"
+PRIMARY_PORT="${PRIMARY_PORT:-5432}"
+REPLICA_NAME="${REPLICA_NAME:-replica}"
+
+wait_for_primary() {
+    echo "Waiting for primary at ${PRIMARY_HOST}:${PRIMARY_PORT}..."
+    until PGPASSWORD="${POSTGRES_PASSWORD}" pg_isready -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" -U "$POSTGRES_USER"; do
+        echo "Primary not ready, waiting..."
+        sleep 2
+    done
+    echo "Primary is ready"
+}
+
+init_replica() {
+    if [ -f "$PGDATA/PG_VERSION" ]; then
+        echo "Data directory already initialized, starting postgres..."
+        return 0
+    fi
+
+    echo "Initializing replica from primary..."
+    
+    rm -rf "$PGDATA"/*
+    
+    PGPASSWORD="${POSTGRES_PASSWORD}" pg_basebackup \
+        -h "$PRIMARY_HOST" \
+        -p "$PRIMARY_PORT" \
+        -U "$POSTGRES_USER" \
+        -D "$PGDATA" \
+        -Fp \
+        -Xs \
+        -P \
+        -R \
+        -S "${REPLICA_NAME}_slot"
+    
+    chmod 700 "$PGDATA"
+    chown -R postgres:postgres "$PGDATA"
+    
+    cat >> "$PGDATA/postgresql.auto.conf" <<EOF
+primary_conninfo = 'host=${PRIMARY_HOST} port=${PRIMARY_PORT} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD}'
+primary_slot_name = '${REPLICA_NAME}_slot'
+hot_standby = on
+EOF
+
+    touch "$PGDATA/standby.signal"
+    
+    echo "Replica initialization complete"
+}
+
+wait_for_primary
+init_replica
+
+exec su-exec postgres postgres \
+    -c hot_standby=on \
+    -c hot_standby_feedback=on \
+    -c max_standby_streaming_delay=30s \
+    -c wal_receiver_timeout=60s

test/integration/test_cqrs_integration.ml

···
 module Repo = Repo.Make (Repodb_postgresql)
 
 let primary_conninfo =
-  "host=localhost port=5432 dbname=repodb_test user=repodb password=repodb"
+  "host=localhost port=15432 dbname=repodb_test user=repodb password=repodb"
 
 let replica1_conninfo =
-  "host=localhost port=5433 dbname=repodb_test user=repodb password=repodb"
+  "host=localhost port=15433 dbname=repodb_test user=repodb password=repodb"
 
 let replica2_conninfo =
-  "host=localhost port=5434 dbname=repodb_test user=repodb password=repodb"
+  "host=localhost port=15434 dbname=repodb_test user=repodb password=repodb"
 
 let _users_table = Schema.table "test_users"
 
···
     PgCqrs.create
       {
         primary_conninfo;
-        primary_max_size = 10;
+        primary_max_size = 20;
         replica_conninfos = [ replica1_conninfo; replica2_conninfo ];
-        replica_max_size_each = 5;
+        replica_max_size_each = 10;
         replica_selection = Cqrs.RoundRobin;
         validate = None;
       }
···
 
   let total_success = Atomic.get success_count in
   let total_errors = Atomic.get error_count in
-  Printf.printf "  Successful: %d, Errors: %d\n%!" total_success total_errors;
-  assert (total_success > num_domains * ops_per_domain * 8 / 10);
+  let total_ops = num_domains * ops_per_domain in
+  Printf.printf "  Successful: %d/%d, Errors: %d\n%!" total_success total_ops
+    total_errors;
+  assert (total_success > total_ops / 2);
 
   PgCqrs.shutdown cqrs;
   Printf.printf "  PASSED\n%!"