name: AgentBeats Leaderboard Sync on: push: branches: [main] paths: - "tasks/**" - "tasks-extra/**" - "integrations/agentbeats/task_sets/**" - "skillsbench_agentbeats/config.py" - "skillsbench_agentbeats/task_sets.py" - ".github/workflows/agentbeats-leaderboard-sync.yml" workflow_dispatch: inputs: target_task_set: description: "New/candidate leaderboard task set to generate; standard-v1 is intentionally blocked" default: "skillsbench-v1.1" type: string leaderboard_repo: description: "Leaderboard repository to receive the generated task set PR" default: "benchflow-ai/skillsbench-leaderboard" type: string leaderboard_base_branch: description: "Base branch for the leaderboard PR" default: "main" type: string include_prebuilt_images: description: "Include a prebuilt_images map from a completed AgentBeats Task Env Images artifact" default: false type: boolean task_env_artifact_run_id: description: "Workflow run id that produced agentbeats-prebuilt-images-; required when include_prebuilt_images is true" default: "" type: string prebuilt_source_task_set: description: "Artifact suffix from the task-env image workflow" default: "skillsbench-v1.1" type: string permissions: contents: read actions: read jobs: sync: name: Generate leaderboard sync PR runs-on: ubuntu-latest env: PYTHONUNBUFFERED: "1" steps: - name: Resolve sync settings run: | set -euo pipefail if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then target_task_set="${{ inputs.target_task_set }}" leaderboard_repo="${{ inputs.leaderboard_repo }}" leaderboard_base="${{ inputs.leaderboard_base_branch }}" include_prebuilt_images="${{ inputs.include_prebuilt_images }}" task_env_artifact_run_id="${{ inputs.task_env_artifact_run_id }}" prebuilt_source_task_set="${{ inputs.prebuilt_source_task_set }}" else target_task_set="skillsbench-v1.1" leaderboard_repo="benchflow-ai/skillsbench-leaderboard" leaderboard_base="main" include_prebuilt_images="false" task_env_artifact_run_id="" prebuilt_source_task_set="skillsbench-v1.1" fi if [[ ! "${target_task_set}" =~ ^[A-Za-z0-9_.-]+$ ]]; then echo "::error::target_task_set must be a direct manifest name" exit 1 fi if [[ "${target_task_set}" == "standard-v1" ]]; then echo "::error::standard-v1 is frozen; create a new skillsbench-v* version for review" exit 1 fi if [[ ! "${leaderboard_repo}" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then echo "::error::leaderboard_repo must be owner/name" exit 1 fi if [[ "${include_prebuilt_images}" == "true" && -z "${task_env_artifact_run_id}" ]]; then echo "::error::task_env_artifact_run_id is required when include_prebuilt_images is true" exit 1 fi { echo "TARGET_TASK_SET=${target_task_set}" echo "LEADERBOARD_REPO=${leaderboard_repo}" echo "LEADERBOARD_BASE=${leaderboard_base}" echo "INCLUDE_PREBUILT_IMAGES=${include_prebuilt_images}" echo "TASK_ENV_ARTIFACT_RUN_ID=${task_env_artifact_run_id}" echo "PREBUILT_SOURCE_TASK_SET=${prebuilt_source_task_set}" } >> "${GITHUB_ENV}" - name: Require leaderboard write token env: LEADERBOARD_SYNC_TOKEN: ${{ secrets.LEADERBOARD_SYNC_TOKEN }} run: | set -euo pipefail if [[ -z "${LEADERBOARD_SYNC_TOKEN}" ]]; then echo "::error::Set LEADERBOARD_SYNC_TOKEN with contents/pull-requests write access to ${LEADERBOARD_REPO}" exit 1 fi - name: Checkout SkillsBench source uses: actions/checkout@v4 with: path: skillsbench - uses: actions/setup-python@v5 with: python-version: "3.12" - name: Install manifest generator dependencies working-directory: skillsbench run: | set -euo pipefail python -m pip install --upgrade pip python -m pip install "pydantic>=2" "pyyaml>=6.0.0" - name: Generate candidate task-set manifest working-directory: skillsbench env: TARGET_TASK_SET: ${{ env.TARGET_TASK_SET }} run: | set -euo pipefail mkdir -p "${RUNNER_TEMP}/agentbeats-sync/task_sets" python -m skillsbench_agentbeats.task_sets \ --task-set "${TARGET_TASK_SET}" \ --output "${RUNNER_TEMP}/agentbeats-sync/task_sets/${TARGET_TASK_SET}.json" - name: Verify candidate task-set manifest env: TARGET_TASK_SET: ${{ env.TARGET_TASK_SET }} TASK_SET_PATH: ${{ runner.temp }}/agentbeats-sync/task_sets/${{ env.TARGET_TASK_SET }}.json run: | set -euo pipefail python - <<'PY' import json import os import re from pathlib import Path path = Path(os.environ["TASK_SET_PATH"]) data = json.loads(path.read_text()) if data.get("schema_version") != "skillsbench.agentbeats.task_set.v1": raise SystemExit("unexpected task-set schema_version") if data.get("task_set") != os.environ["TARGET_TASK_SET"]: raise SystemExit("manifest task_set does not match target") if data.get("task_set") == "standard-v1": raise SystemExit("standard-v1 must not be generated by this sync workflow") tasks = data.get("tasks") if not isinstance(tasks, list) or not tasks: raise SystemExit("manifest must contain tasks") if data.get("task_count") != len(tasks): raise SystemExit("task_count does not match tasks length") seen = set() for task in tasks: task_id = task.get("task_id") if isinstance(task, dict) else None if not isinstance(task_id, str) or not re.fullmatch(r"[^/\\]+", task_id): raise SystemExit(f"invalid task_id: {task_id!r}") if task_id in seen: raise SystemExit(f"duplicate task_id: {task_id}") seen.add(task_id) digest = task.get("task_digest") if not isinstance(digest, str) or not digest.startswith("sha256:"): raise SystemExit(f"missing task digest for {task_id}") print(f"verified {len(tasks)} tasks for {data['task_set']} ({data['task_set_digest']})") PY - name: Download optional prebuilt image map artifact if: env.INCLUDE_PREBUILT_IMAGES == 'true' uses: actions/download-artifact@v4 with: name: agentbeats-prebuilt-images-${{ env.PREBUILT_SOURCE_TASK_SET }} run-id: ${{ env.TASK_ENV_ARTIFACT_RUN_ID }} path: ${{ runner.temp }}/agentbeats-sync/prebuilt github-token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout leaderboard repository uses: actions/checkout@v4 with: repository: ${{ env.LEADERBOARD_REPO }} ref: ${{ env.LEADERBOARD_BASE }} token: ${{ secrets.LEADERBOARD_SYNC_TOKEN }} path: leaderboard - name: Copy generated leaderboard files env: TARGET_TASK_SET: ${{ env.TARGET_TASK_SET }} INCLUDE_PREBUILT_IMAGES: ${{ env.INCLUDE_PREBUILT_IMAGES }} TASK_SET_PATH: ${{ runner.temp }}/agentbeats-sync/task_sets/${{ env.TARGET_TASK_SET }}.json SOURCE_REPOSITORY: ${{ github.repository }} SOURCE_REF: ${{ github.ref_name }} SOURCE_SHA: ${{ github.sha }} WORKFLOW_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail mkdir -p leaderboard/task_sets leaderboard/prebuilt_images cp "${TASK_SET_PATH}" "leaderboard/task_sets/${TARGET_TASK_SET}.json" python - <<'PY' import json import os from pathlib import Path target = os.environ["TARGET_TASK_SET"] manifest = json.loads(Path(os.environ["TASK_SET_PATH"]).read_text()) metadata = { "schema_version": "skillsbench.agentbeats.sync_source.v1", "task_set": target, "source_repository": os.environ["SOURCE_REPOSITORY"], "source_ref": os.environ["SOURCE_REF"], "source_sha": os.environ["SOURCE_SHA"], "workflow_run_url": os.environ["WORKFLOW_RUN_URL"], "task_count": manifest["task_count"], "task_set_digest": manifest["task_set_digest"], "prebuilt_images_included": os.environ["INCLUDE_PREBUILT_IMAGES"] == "true", "review_required": True, "active_public_task_sets_mutated": False, } out = Path("leaderboard") / "task_sets" / f"{target}.source.json" out.write_text(json.dumps(metadata, indent=2) + "\n") PY if [[ "${INCLUDE_PREBUILT_IMAGES}" == "true" ]]; then map_path="$(find "${RUNNER_TEMP}/agentbeats-sync/prebuilt" -type f -name '*.json' | head -n 1)" if [[ -z "${map_path}" ]]; then echo "::error::No JSON prebuilt image artifact found" exit 1 fi cp "${map_path}" "leaderboard/prebuilt_images/${TARGET_TASK_SET}.json" fi - name: Verify optional prebuilt image map matches task set if: env.INCLUDE_PREBUILT_IMAGES == 'true' env: TARGET_TASK_SET: ${{ env.TARGET_TASK_SET }} run: | set -euo pipefail python - <<'PY' import json import os import re from pathlib import Path target = os.environ["TARGET_TASK_SET"] manifest = json.loads((Path("leaderboard") / "task_sets" / f"{target}.json").read_text()) images = json.loads((Path("leaderboard") / "prebuilt_images" / f"{target}.json").read_text()) task_ids = {task["task_id"] for task in manifest["tasks"]} image_ids = set(images) missing = sorted(task_ids - image_ids) extra = sorted(image_ids - task_ids) bad_refs = sorted(task_id for task_id, image in images.items() if not isinstance(image, str) or re.search(r"@sha256:[0-9a-fA-F]{64}$", image) is None) if missing or extra or bad_refs: raise SystemExit( "prebuilt image map mismatch: " f"missing={missing[:10]} extra={extra[:10]} bad_refs={bad_refs[:10]}" ) print(f"verified {len(images)} digest-pinned prebuilt images for {target}") PY - name: Open or update leaderboard PR uses: peter-evans/create-pull-request@v6 with: path: leaderboard token: ${{ secrets.LEADERBOARD_SYNC_TOKEN }} branch: agentbeats/sync-${{ env.TARGET_TASK_SET }} base: ${{ env.LEADERBOARD_BASE }} delete-branch: false commit-message: Sync SkillsBench AgentBeats ${{ env.TARGET_TASK_SET }} title: Sync SkillsBench AgentBeats ${{ env.TARGET_TASK_SET }} body: | Review-gated AgentBeats task-set sync from `${{ github.repository }}@${{ github.sha }}`. - Target task set: `${{ env.TARGET_TASK_SET }}` - Active public `standard-v1` is not mutated by this workflow. - Prebuilt image map included: `${{ env.INCLUDE_PREBUILT_IMAGES }}` - Source workflow run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} Before promoting this task set for public AgentBeats use, verify the corresponding task environment images are digest-pinned and run a representative smoke/shard. labels: | agentbeats automated-sync