285 lines
12 KiBLFS
YAML
285 lines
12 KiBLFS
YAML
name: AgentBeats Leaderboard Sync
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- "tasks/**"
|
|
- "tasks-extra/**"
|
|
- "integrations/agentbeats/task_sets/**"
|
|
- "skillsbench_agentbeats/config.py"
|
|
- "skillsbench_agentbeats/task_sets.py"
|
|
- ".github/workflows/agentbeats-leaderboard-sync.yml"
|
|
workflow_dispatch:
|
|
inputs:
|
|
target_task_set:
|
|
description: "New/candidate leaderboard task set to generate; standard-v1 is intentionally blocked"
|
|
default: "skillsbench-v1.1"
|
|
type: string
|
|
leaderboard_repo:
|
|
description: "Leaderboard repository to receive the generated task set PR"
|
|
default: "benchflow-ai/skillsbench-leaderboard"
|
|
type: string
|
|
leaderboard_base_branch:
|
|
description: "Base branch for the leaderboard PR"
|
|
default: "main"
|
|
type: string
|
|
include_prebuilt_images:
|
|
description: "Include a prebuilt_images map from a completed AgentBeats Task Env Images artifact"
|
|
default: false
|
|
type: boolean
|
|
task_env_artifact_run_id:
|
|
description: "Workflow run id that produced agentbeats-prebuilt-images-<source_task_set>; required when include_prebuilt_images is true"
|
|
default: ""
|
|
type: string
|
|
prebuilt_source_task_set:
|
|
description: "Artifact suffix from the task-env image workflow"
|
|
default: "skillsbench-v1.1"
|
|
type: string
|
|
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
|
|
jobs:
|
|
sync:
|
|
name: Generate leaderboard sync PR
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
PYTHONUNBUFFERED: "1"
|
|
steps:
|
|
- name: Resolve sync settings
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
|
target_task_set="${{ inputs.target_task_set }}"
|
|
leaderboard_repo="${{ inputs.leaderboard_repo }}"
|
|
leaderboard_base="${{ inputs.leaderboard_base_branch }}"
|
|
include_prebuilt_images="${{ inputs.include_prebuilt_images }}"
|
|
task_env_artifact_run_id="${{ inputs.task_env_artifact_run_id }}"
|
|
prebuilt_source_task_set="${{ inputs.prebuilt_source_task_set }}"
|
|
else
|
|
target_task_set="skillsbench-v1.1"
|
|
leaderboard_repo="benchflow-ai/skillsbench-leaderboard"
|
|
leaderboard_base="main"
|
|
include_prebuilt_images="false"
|
|
task_env_artifact_run_id=""
|
|
prebuilt_source_task_set="skillsbench-v1.1"
|
|
fi
|
|
|
|
if [[ ! "${target_task_set}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
|
|
echo "::error::target_task_set must be a direct manifest name"
|
|
exit 1
|
|
fi
|
|
if [[ "${target_task_set}" == "standard-v1" ]]; then
|
|
echo "::error::standard-v1 is frozen; create a new skillsbench-v* version for review"
|
|
exit 1
|
|
fi
|
|
if [[ ! "${leaderboard_repo}" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then
|
|
echo "::error::leaderboard_repo must be owner/name"
|
|
exit 1
|
|
fi
|
|
if [[ "${include_prebuilt_images}" == "true" && -z "${task_env_artifact_run_id}" ]]; then
|
|
echo "::error::task_env_artifact_run_id is required when include_prebuilt_images is true"
|
|
exit 1
|
|
fi
|
|
|
|
{
|
|
echo "TARGET_TASK_SET=${target_task_set}"
|
|
echo "LEADERBOARD_REPO=${leaderboard_repo}"
|
|
echo "LEADERBOARD_BASE=${leaderboard_base}"
|
|
echo "INCLUDE_PREBUILT_IMAGES=${include_prebuilt_images}"
|
|
echo "TASK_ENV_ARTIFACT_RUN_ID=${task_env_artifact_run_id}"
|
|
echo "PREBUILT_SOURCE_TASK_SET=${prebuilt_source_task_set}"
|
|
} >> "${GITHUB_ENV}"
|
|
|
|
- name: Require leaderboard write token
|
|
env:
|
|
LEADERBOARD_SYNC_TOKEN: ${{ secrets.LEADERBOARD_SYNC_TOKEN }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ -z "${LEADERBOARD_SYNC_TOKEN}" ]]; then
|
|
echo "::error::Set LEADERBOARD_SYNC_TOKEN with contents/pull-requests write access to ${LEADERBOARD_REPO}"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Checkout SkillsBench source
|
|
uses: actions/checkout@v4
|
|
with:
|
|
path: skillsbench
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Install manifest generator dependencies
|
|
working-directory: skillsbench
|
|
run: |
|
|
set -euo pipefail
|
|
python -m pip install --upgrade pip
|
|
python -m pip install "pydantic>=2" "pyyaml>=6.0.0"
|
|
|
|
- name: Generate candidate task-set manifest
|
|
working-directory: skillsbench
|
|
env:
|
|
TARGET_TASK_SET: ${{ env.TARGET_TASK_SET }}
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "${RUNNER_TEMP}/agentbeats-sync/task_sets"
|
|
python -m skillsbench_agentbeats.task_sets \
|
|
--task-set "${TARGET_TASK_SET}" \
|
|
--output "${RUNNER_TEMP}/agentbeats-sync/task_sets/${TARGET_TASK_SET}.json"
|
|
|
|
- name: Verify candidate task-set manifest
|
|
env:
|
|
TARGET_TASK_SET: ${{ env.TARGET_TASK_SET }}
|
|
TASK_SET_PATH: ${{ runner.temp }}/agentbeats-sync/task_sets/${{ env.TARGET_TASK_SET }}.json
|
|
run: |
|
|
set -euo pipefail
|
|
python - <<'PY'
|
|
import json
|
|
import os
|
|
import re
|
|
from pathlib import Path
|
|
|
|
path = Path(os.environ["TASK_SET_PATH"])
|
|
data = json.loads(path.read_text())
|
|
if data.get("schema_version") != "skillsbench.agentbeats.task_set.v1":
|
|
raise SystemExit("unexpected task-set schema_version")
|
|
if data.get("task_set") != os.environ["TARGET_TASK_SET"]:
|
|
raise SystemExit("manifest task_set does not match target")
|
|
if data.get("task_set") == "standard-v1":
|
|
raise SystemExit("standard-v1 must not be generated by this sync workflow")
|
|
tasks = data.get("tasks")
|
|
if not isinstance(tasks, list) or not tasks:
|
|
raise SystemExit("manifest must contain tasks")
|
|
if data.get("task_count") != len(tasks):
|
|
raise SystemExit("task_count does not match tasks length")
|
|
seen = set()
|
|
for task in tasks:
|
|
task_id = task.get("task_id") if isinstance(task, dict) else None
|
|
if not isinstance(task_id, str) or not re.fullmatch(r"[^/\\]+", task_id):
|
|
raise SystemExit(f"invalid task_id: {task_id!r}")
|
|
if task_id in seen:
|
|
raise SystemExit(f"duplicate task_id: {task_id}")
|
|
seen.add(task_id)
|
|
digest = task.get("task_digest")
|
|
if not isinstance(digest, str) or not digest.startswith("sha256:"):
|
|
raise SystemExit(f"missing task digest for {task_id}")
|
|
print(f"verified {len(tasks)} tasks for {data['task_set']} ({data['task_set_digest']})")
|
|
PY
|
|
|
|
- name: Download optional prebuilt image map artifact
|
|
if: env.INCLUDE_PREBUILT_IMAGES == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: agentbeats-prebuilt-images-${{ env.PREBUILT_SOURCE_TASK_SET }}
|
|
run-id: ${{ env.TASK_ENV_ARTIFACT_RUN_ID }}
|
|
path: ${{ runner.temp }}/agentbeats-sync/prebuilt
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Checkout leaderboard repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: ${{ env.LEADERBOARD_REPO }}
|
|
ref: ${{ env.LEADERBOARD_BASE }}
|
|
token: ${{ secrets.LEADERBOARD_SYNC_TOKEN }}
|
|
path: leaderboard
|
|
|
|
- name: Copy generated leaderboard files
|
|
env:
|
|
TARGET_TASK_SET: ${{ env.TARGET_TASK_SET }}
|
|
INCLUDE_PREBUILT_IMAGES: ${{ env.INCLUDE_PREBUILT_IMAGES }}
|
|
TASK_SET_PATH: ${{ runner.temp }}/agentbeats-sync/task_sets/${{ env.TARGET_TASK_SET }}.json
|
|
SOURCE_REPOSITORY: ${{ github.repository }}
|
|
SOURCE_REF: ${{ github.ref_name }}
|
|
SOURCE_SHA: ${{ github.sha }}
|
|
WORKFLOW_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p leaderboard/task_sets leaderboard/prebuilt_images
|
|
cp "${TASK_SET_PATH}" "leaderboard/task_sets/${TARGET_TASK_SET}.json"
|
|
python - <<'PY'
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
target = os.environ["TARGET_TASK_SET"]
|
|
manifest = json.loads(Path(os.environ["TASK_SET_PATH"]).read_text())
|
|
metadata = {
|
|
"schema_version": "skillsbench.agentbeats.sync_source.v1",
|
|
"task_set": target,
|
|
"source_repository": os.environ["SOURCE_REPOSITORY"],
|
|
"source_ref": os.environ["SOURCE_REF"],
|
|
"source_sha": os.environ["SOURCE_SHA"],
|
|
"workflow_run_url": os.environ["WORKFLOW_RUN_URL"],
|
|
"task_count": manifest["task_count"],
|
|
"task_set_digest": manifest["task_set_digest"],
|
|
"prebuilt_images_included": os.environ["INCLUDE_PREBUILT_IMAGES"] == "true",
|
|
"review_required": True,
|
|
"active_public_task_sets_mutated": False,
|
|
}
|
|
out = Path("leaderboard") / "task_sets" / f"{target}.source.json"
|
|
out.write_text(json.dumps(metadata, indent=2) + "\n")
|
|
PY
|
|
|
|
if [[ "${INCLUDE_PREBUILT_IMAGES}" == "true" ]]; then
|
|
map_path="$(find "${RUNNER_TEMP}/agentbeats-sync/prebuilt" -type f -name '*.json' | head -n 1)"
|
|
if [[ -z "${map_path}" ]]; then
|
|
echo "::error::No JSON prebuilt image artifact found"
|
|
exit 1
|
|
fi
|
|
cp "${map_path}" "leaderboard/prebuilt_images/${TARGET_TASK_SET}.json"
|
|
fi
|
|
|
|
- name: Verify optional prebuilt image map matches task set
|
|
if: env.INCLUDE_PREBUILT_IMAGES == 'true'
|
|
env:
|
|
TARGET_TASK_SET: ${{ env.TARGET_TASK_SET }}
|
|
run: |
|
|
set -euo pipefail
|
|
python - <<'PY'
|
|
import json
|
|
import os
|
|
import re
|
|
from pathlib import Path
|
|
|
|
target = os.environ["TARGET_TASK_SET"]
|
|
manifest = json.loads((Path("leaderboard") / "task_sets" / f"{target}.json").read_text())
|
|
images = json.loads((Path("leaderboard") / "prebuilt_images" / f"{target}.json").read_text())
|
|
task_ids = {task["task_id"] for task in manifest["tasks"]}
|
|
image_ids = set(images)
|
|
missing = sorted(task_ids - image_ids)
|
|
extra = sorted(image_ids - task_ids)
|
|
bad_refs = sorted(task_id for task_id, image in images.items() if not isinstance(image, str) or re.search(r"@sha256:[0-9a-fA-F]{64}$", image) is None)
|
|
if missing or extra or bad_refs:
|
|
raise SystemExit(
|
|
"prebuilt image map mismatch: "
|
|
f"missing={missing[:10]} extra={extra[:10]} bad_refs={bad_refs[:10]}"
|
|
)
|
|
print(f"verified {len(images)} digest-pinned prebuilt images for {target}")
|
|
PY
|
|
|
|
- name: Open or update leaderboard PR
|
|
uses: peter-evans/create-pull-request@v6
|
|
with:
|
|
path: leaderboard
|
|
token: ${{ secrets.LEADERBOARD_SYNC_TOKEN }}
|
|
branch: agentbeats/sync-${{ env.TARGET_TASK_SET }}
|
|
base: ${{ env.LEADERBOARD_BASE }}
|
|
delete-branch: false
|
|
commit-message: Sync SkillsBench AgentBeats ${{ env.TARGET_TASK_SET }}
|
|
title: Sync SkillsBench AgentBeats ${{ env.TARGET_TASK_SET }}
|
|
body: |
|
|
Review-gated AgentBeats task-set sync from `${{ github.repository }}@${{ github.sha }}`.
|
|
|
|
- Target task set: `${{ env.TARGET_TASK_SET }}`
|
|
- Active public `standard-v1` is not mutated by this workflow.
|
|
- Prebuilt image map included: `${{ env.INCLUDE_PREBUILT_IMAGES }}`
|
|
- Source workflow run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
|
|
Before promoting this task set for public AgentBeats use, verify the corresponding task environment images are digest-pinned and run a representative smoke/shard.
|
|
labels: |
|
|
agentbeats
|
|
automated-sync
|