68 lines
3.3 KiB
Bash
68 lines
3.3 KiB
Bash
#!/usr/bin/env bash
|
|
# Container output verification and bounded artifact capture.
|
|
|
|
verify_output() {
|
|
local run_root=$1 run_id=$2 log_dir="$1/verifier"
|
|
local verifier_started verifier_ended verifier_wall docker_exit reward verification_status description
|
|
mkdir -p "$log_dir"
|
|
# BenchFlow's native task.md verifier contract: upload verifier/ to
|
|
# /verifier, then expose the legacy /tests path as a symlink only if the
|
|
# image has not already provided real /tests content. Do not overwrite that
|
|
# content; older verifier scripts may legitimately depend on it.
|
|
docker exec "$run_id" mkdir -p /verifier /logs/verifier
|
|
docker cp "$VERIFIER_SOURCE/." "$run_id:/verifier"
|
|
docker exec "$run_id" bash -lc '[ -e /tests ] || ln -s /verifier /tests'
|
|
docker exec "$run_id" chmod +x /verifier/test.sh
|
|
verifier_started=$(now_ms)
|
|
if timeout --foreground --signal=INT --kill-after=30s "${VERIFIER_TIMEOUT_SECONDS}s" \
|
|
docker exec "${VERIFIER_ENV_ARGS[@]}" "$run_id" /verifier/test.sh > "$log_dir/verifier.stdout.log" 2>&1; then
|
|
docker_exit=0
|
|
else
|
|
docker_exit=$?
|
|
fi
|
|
docker cp "$run_id:/logs/verifier/." "$log_dir" >/dev/null 2>&1 || true
|
|
verifier_ended=$(now_ms)
|
|
verifier_wall=$((verifier_ended - verifier_started))
|
|
reward=""
|
|
[ ! -f "$log_dir/reward.txt" ] || reward=$(tr -d '[:space:]' < "$log_dir/reward.txt")
|
|
if [ "$docker_exit" -eq 0 ] && [ "$reward" = 1 ]; then
|
|
verification_status=passed
|
|
description=none
|
|
else
|
|
verification_status=failed
|
|
description="Verifier exited with code ${docker_exit} and wrote reward=${reward:-missing}. See verifier.stdout.log for details."
|
|
fi
|
|
{
|
|
printf 'docker_exit_code=%s\nreward=%s\nverifier_wall_ms=%s\n' "$docker_exit" "$reward" "$verifier_wall"
|
|
printf 'verifier_log=%s\nverification_status=%s\nproblem_description=%s\n' "$log_dir/verifier.stdout.log" "$verification_status" "$description"
|
|
} > "$log_dir/summary.env"
|
|
[ "$verification_status" = passed ] && return 0
|
|
printf 'VERIFIER_ISSUE: %s\n' "$description" >&2
|
|
return 1
|
|
}
|
|
|
|
capture_agent_artifacts() {
|
|
local run_root=$1 run_id=$2 diff_path="$1/container-diff.txt"
|
|
local artifact_root="$1/artifacts" status container_path relative_path size destination
|
|
mkdir -p "$artifact_root"
|
|
while IFS=' ' read -r status container_path; do
|
|
[ "$status" = A ] || [ "$status" = C ] || continue
|
|
# Copy only modest, task-created outputs. Package caches and the mounted
|
|
# workspace are inputs/ephemera, never benchmark artifacts.
|
|
case "$container_path" in
|
|
/root/.cache/*|/root/.local/*|/root/.m2/*|/home/*/.cache/*|/home/*/.local/*|/home/*/.m2/*|*/.git/*|/etc/*|/opt/*|/tmp/*|/usr/*|/var/*|/workspace/*) continue ;;
|
|
esac
|
|
docker exec "$run_id" test -f "$container_path" >/dev/null 2>&1 || continue
|
|
size=$(docker exec "$run_id" stat -c '%s' "$container_path" 2>/dev/null || printf '0')
|
|
[[ "$size" =~ ^[0-9]+$ ]] || continue
|
|
[ "$size" -le 20971520 ] || continue
|
|
relative_path=${container_path#/}
|
|
destination="$artifact_root/$relative_path"
|
|
mkdir -p "$(dirname "$destination")"
|
|
docker cp "$run_id:$container_path" "$destination" >/dev/null
|
|
done < "$diff_path"
|
|
if find "$artifact_root" -type f -print -quit | grep -q .; then
|
|
find "$artifact_root" -type f -print0 | sort -z | xargs -0 sha256sum > "$run_root/output-sha256.txt"
|
|
fi
|
|
}
|