Initial commit

2026-09-04 14:58:42 +08:00
commit 439cad87d9
4601 changed files with 29440 additions and 0 deletions
@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from ..models import RolloutTrace
+from ..scoring.state import acp_events_to_state, read_acp_events
+
+
+VARIANT_TO_COMPILE_TYPE = {
+    "model_skill": "model_compile",
+    "ori_skill": "ori",
+}
+
+
+def event_to_state(trajectory_path: Path) -> tuple[list[dict[str, str]], bool]:
+    events = read_acp_events(trajectory_path)
+    skill_invoked = any(
+        event.get("type") == "tool_call"
+        and any(
+            str(event.get(field, "")).strip().lower() == "skill"
+            for field in ("title", "kind")
+        )
+        for event in events
+    )
+    return acp_events_to_state(events), skill_invoked
+
+
+def trajectory_for_test(test_dir: Path) -> Path | None:
+    candidates = sorted(test_dir.rglob("acp_trajectory.jsonl"))
+    if not candidates:
+        return None
+    canonical = [path for path in candidates if "trajectory" in path.parts]
+    return canonical[0] if canonical else candidates[0]
+
+
+def _result_for_trajectory(trajectory_path: Path) -> dict[str, Any]:
+    result_path = trajectory_path.parent.parent / "result.json"
+    if not result_path.is_file():
+        raise ValueError(f"missing structured BenchFlow result: {result_path}")
+    value = json.loads(result_path.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise ValueError(f"BenchFlow result must be a JSON object: {result_path}")
+    return value
+
+
+def load_benchflow_trace(
+    test_dir: Path,
+    task_name: str,
+    compile_type: str,
+) -> RolloutTrace:
+    trajectory = trajectory_for_test(test_dir)
+    if trajectory is None:
+        raise ValueError(f"missing acp_trajectory.jsonl under {test_dir}")
+    result = _result_for_trajectory(trajectory)
+    agent_timeout = result.get("agent_timeout_info")
+    idle_timeout = result.get("idle_timeout_info")
+    timeout_info = (
+        agent_timeout
+        if isinstance(agent_timeout, dict)
+        else idle_timeout if isinstance(idle_timeout, dict) else None
+    )
+    timed_out = timeout_info is not None
+    metadata: dict[str, Any] = {
+        "source": str(test_dir.resolve()),
+        "termination": "timeout" if timed_out else "completed",
+    }
+    timing = result.get("timing") if isinstance(result.get("timing"), dict) else {}
+    execution_seconds = timing.get("agent_execution")
+    if execution_seconds is None and isinstance(timeout_info, dict):
+        execution_seconds = timeout_info.get(
+            "wall_clock_elapsed_sec", timeout_info.get("timeout_sec")
+        )
+    if isinstance(execution_seconds, (int, float)):
+        metadata["agent_execution_seconds"] = float(execution_seconds)
+    if isinstance(result.get("n_tool_calls"), int):
+        metadata["tool_calls"] = result["n_tool_calls"]
+    if timed_out:
+        metadata["timeout_reason"] = timeout_info.get("reason")
+        metadata["timeout_seconds"] = timeout_info.get(
+            "timeout_sec", timeout_info.get("idle_timeout_sec")
+        )
+        metadata["partial_trajectory"] = bool(result.get("partial_trajectory", False))
+        metadata["error_category"] = result.get("error_category")
+    state, skill_invoked = event_to_state(trajectory)
+    return RolloutTrace(
+        trace_id=f"{task_name}/{compile_type}/{test_dir.name}",
+        task_name=task_name,
+        compile_type=compile_type,
+        test_name=test_dir.name,
+        state=state,
+        skill_invoked=skill_invoked,
+        timed_out=timed_out,
+        metadata=metadata,
+    )
+
+
+def _variant_dirs(input_path: Path) -> list[tuple[Path, str, str]]:
+    if input_path.name in VARIANT_TO_COMPILE_TYPE:
+        return [(
+            input_path,
+            input_path.parent.name,
+            VARIANT_TO_COMPILE_TYPE[input_path.name],
+        )]
+    direct_variants = [
+        (input_path / variant_name, input_path.name, compile_type)
+        for variant_name, compile_type in VARIANT_TO_COMPILE_TYPE.items()
+        if (input_path / variant_name).is_dir()
+    ]
+    if direct_variants:
+        return direct_variants
+    variants = []
+    for task_dir in sorted(path for path in input_path.iterdir() if path.is_dir()):
+        for variant_name, compile_type in VARIANT_TO_COMPILE_TYPE.items():
+            variant_dir = task_dir / variant_name
+            if variant_dir.is_dir():
+                variants.append((variant_dir, task_dir.name, compile_type))
+    return variants
+
+
+def load_benchflow_traces(input_path: Path) -> list[RolloutTrace]:
+    input_path = input_path.resolve()
+    if not input_path.is_dir():
+        raise ValueError(f"BenchFlow input does not exist: {input_path}")
+    variants = _variant_dirs(input_path)
+    if not variants:
+        raise ValueError(f"no model_skill or ori_skill directories under {input_path}")
+    traces = []
+    for variant_dir, task_name, compile_type in variants:
+        for test_dir in sorted(path for path in variant_dir.glob("test-*") if path.is_dir()):
+            traces.append(load_benchflow_trace(test_dir, task_name, compile_type))
+    return traces