Initial commit
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
from .cli import main
|
||||
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,300 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .models import Patch, RolloutTrace
|
||||
from .scoring.pre_score import PRE_SCORE_VERSION, PreScoreResult
|
||||
from .storage import (
|
||||
atomic_write_json,
|
||||
load_json,
|
||||
package_hash,
|
||||
read_jsonl,
|
||||
sha256_file,
|
||||
sha256_text,
|
||||
)
|
||||
|
||||
|
||||
CACHE_SCHEMA_VERSION = 1
|
||||
CACHE_PRODUCER = "dynamic-compile-fast"
|
||||
SCORE_ARTIFACTS = (
|
||||
"pre_scores.jsonl",
|
||||
"agentrm_scores.jsonl",
|
||||
"effective_scores.jsonl",
|
||||
)
|
||||
|
||||
|
||||
def fingerprint(value: Any) -> str:
|
||||
payload = json.dumps(
|
||||
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||
)
|
||||
return sha256_text(payload)
|
||||
|
||||
|
||||
def trace_fingerprint(traces: Iterable[RolloutTrace]) -> str:
|
||||
return fingerprint([asdict(trace) for trace in traces])
|
||||
|
||||
|
||||
def _artifact_hash(path: Path) -> str:
|
||||
if path.is_file():
|
||||
return sha256_file(path)
|
||||
if path.is_dir():
|
||||
return package_hash(path)
|
||||
raise OSError(f"cache artifact does not exist: {path}")
|
||||
|
||||
|
||||
def write_manifest(
|
||||
root: Path,
|
||||
name: str,
|
||||
*,
|
||||
stage: str,
|
||||
input_hash: str,
|
||||
config: dict[str, Any],
|
||||
artifacts: Iterable[str],
|
||||
) -> None:
|
||||
artifact_names = tuple(artifacts)
|
||||
hashes = {item: _artifact_hash(root / item) for item in artifact_names}
|
||||
atomic_write_json(
|
||||
root / name,
|
||||
{
|
||||
"schema_version": CACHE_SCHEMA_VERSION,
|
||||
"producer": CACHE_PRODUCER,
|
||||
"stage": stage,
|
||||
"status": "complete",
|
||||
"input_hash": input_hash,
|
||||
"config_hash": fingerprint(config),
|
||||
"artifacts": hashes,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def valid_manifest(
|
||||
root: Path,
|
||||
name: str,
|
||||
*,
|
||||
stage: str,
|
||||
input_hash: str,
|
||||
config: dict[str, Any],
|
||||
artifacts: Iterable[str],
|
||||
) -> bool:
|
||||
try:
|
||||
value = load_json(root / name)
|
||||
expected = tuple(artifacts)
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
if value.get("schema_version") != CACHE_SCHEMA_VERSION:
|
||||
return False
|
||||
if value.get("producer") != CACHE_PRODUCER:
|
||||
return False
|
||||
if value.get("stage") != stage or value.get("status") != "complete":
|
||||
return False
|
||||
if value.get("input_hash") != input_hash:
|
||||
return False
|
||||
if value.get("config_hash") != fingerprint(config):
|
||||
return False
|
||||
hashes = value.get("artifacts")
|
||||
if not isinstance(hashes, dict) or set(hashes) != set(expected):
|
||||
return False
|
||||
return all(
|
||||
isinstance(hashes[item], str)
|
||||
and hashes[item] == _artifact_hash(root / item)
|
||||
for item in expected
|
||||
)
|
||||
except (OSError, TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def load_score_cache(
|
||||
traces: list[RolloutTrace],
|
||||
score_dir: Path,
|
||||
*,
|
||||
input_hash: str,
|
||||
config: dict[str, Any],
|
||||
) -> tuple[dict[str, float], list[dict[str, Any]]] | None:
|
||||
if not valid_manifest(
|
||||
score_dir,
|
||||
".score-cache.json",
|
||||
stage="score",
|
||||
input_hash=input_hash,
|
||||
config=config,
|
||||
artifacts=SCORE_ARTIFACTS,
|
||||
):
|
||||
return None
|
||||
try:
|
||||
pre_rows = read_jsonl(score_dir / "pre_scores.jsonl")
|
||||
agentrm_rows = read_jsonl(score_dir / "agentrm_scores.jsonl")
|
||||
score_rows = read_jsonl(score_dir / "effective_scores.jsonl")
|
||||
trace_by_id = {trace.trace_id: trace for trace in traces}
|
||||
if len(trace_by_id) != len(traces):
|
||||
return None
|
||||
pre_by_id: dict[str, PreScoreResult] = {}
|
||||
for row in pre_rows:
|
||||
result = PreScoreResult.from_dict(row)
|
||||
if result.trace_id in pre_by_id:
|
||||
return None
|
||||
if result.scoring_version != PRE_SCORE_VERSION:
|
||||
return None
|
||||
if result.route not in {"agentrm", "fixed_score"}:
|
||||
return None
|
||||
if result.route == "fixed_score":
|
||||
if result.score is None or not math.isfinite(result.score):
|
||||
return None
|
||||
pre_by_id[result.trace_id] = result
|
||||
if set(pre_by_id) != set(trace_by_id):
|
||||
return None
|
||||
|
||||
expected_agentrm = {
|
||||
trace.key.as_tuple()
|
||||
for trace in traces
|
||||
if pre_by_id[trace.trace_id].route == "agentrm"
|
||||
}
|
||||
actual_agentrm: set[tuple[str, str, str]] = set()
|
||||
for row in agentrm_rows:
|
||||
key = (
|
||||
str(row["task_name"]),
|
||||
str(row["compile_type"]),
|
||||
str(row["test_name"]),
|
||||
)
|
||||
score = float(row["score"])
|
||||
if key in actual_agentrm or not math.isfinite(score):
|
||||
return None
|
||||
if int(row["n_tokens"]) < 0:
|
||||
return None
|
||||
actual_agentrm.add(key)
|
||||
if actual_agentrm != expected_agentrm:
|
||||
return None
|
||||
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
scores: dict[str, float] = {}
|
||||
for row in score_rows:
|
||||
trace_id = str(row["trace_id"])
|
||||
score = float(row["effective_score"])
|
||||
if trace_id in by_id or not math.isfinite(score):
|
||||
return None
|
||||
trace = trace_by_id.get(trace_id)
|
||||
if trace is None or (
|
||||
str(row["task_name"]),
|
||||
str(row["compile_type"]),
|
||||
str(row["test_name"]),
|
||||
) != trace.key.as_tuple():
|
||||
return None
|
||||
expected_source = (
|
||||
"agentrm"
|
||||
if pre_by_id[trace_id].route == "agentrm"
|
||||
else "pre_score"
|
||||
)
|
||||
if row.get("score_source") != expected_source:
|
||||
return None
|
||||
by_id[trace_id] = row
|
||||
scores[trace_id] = score
|
||||
if set(by_id) != set(trace_by_id):
|
||||
return None
|
||||
return scores, [by_id[trace.trace_id] for trace in traces]
|
||||
except (KeyError, OSError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def load_maps_cache(
|
||||
output_dir: Path,
|
||||
selected: list[str],
|
||||
*,
|
||||
input_hash: str,
|
||||
config: dict[str, Any],
|
||||
) -> list[dict[str, Any]] | None:
|
||||
if not valid_manifest(
|
||||
output_dir,
|
||||
".maps-cache.json",
|
||||
stage="maps",
|
||||
input_hash=input_hash,
|
||||
config=config,
|
||||
artifacts=("maps.json",),
|
||||
):
|
||||
return None
|
||||
try:
|
||||
value = load_json(output_dir / "maps.json")
|
||||
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
|
||||
return None
|
||||
by_id = {str(item.get("trace_id", "")): item for item in value}
|
||||
if len(by_id) != len(value) or set(by_id) != set(selected):
|
||||
return None
|
||||
if any(
|
||||
not isinstance(item.get("patterns"), list)
|
||||
or not all(isinstance(pattern, dict) for pattern in item["patterns"])
|
||||
for item in value
|
||||
):
|
||||
return None
|
||||
return [by_id[trace_id] for trace_id in selected]
|
||||
except (OSError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def load_reduction_cache(
|
||||
output_dir: Path,
|
||||
*,
|
||||
input_hash: str,
|
||||
config: dict[str, Any],
|
||||
) -> dict[str, Any] | None:
|
||||
if not valid_manifest(
|
||||
output_dir,
|
||||
".reduction-cache.json",
|
||||
stage="reduction",
|
||||
input_hash=input_hash,
|
||||
config=config,
|
||||
artifacts=("reduction.json",),
|
||||
):
|
||||
return None
|
||||
try:
|
||||
value = load_json(output_dir / "reduction.json")
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
for key in ("successful_pattern", "failure_pattern", "selected_gap"):
|
||||
item = value.get(key)
|
||||
if not isinstance(item, dict) or not str(item.get("description", "")).strip():
|
||||
return None
|
||||
return value
|
||||
except (OSError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def load_candidate_cache(
|
||||
output_dir: Path,
|
||||
candidate_name: str,
|
||||
skill_hash: str,
|
||||
*,
|
||||
input_hash: str,
|
||||
config: dict[str, Any],
|
||||
) -> Path | None:
|
||||
candidate_relative = f"candidate-skill/{candidate_name}"
|
||||
if not valid_manifest(
|
||||
output_dir,
|
||||
".candidate-cache.json",
|
||||
stage="candidate",
|
||||
input_hash=input_hash,
|
||||
config=config,
|
||||
artifacts=("patch.json", candidate_relative),
|
||||
):
|
||||
return None
|
||||
candidate = output_dir / candidate_relative
|
||||
try:
|
||||
value = load_json(output_dir / "patch.json")
|
||||
if not isinstance(value, dict) or value.get("skill_hash") != skill_hash:
|
||||
return None
|
||||
patches = value.get("patches")
|
||||
if not isinstance(patches, list) or not 1 <= len(patches) <= 2:
|
||||
return None
|
||||
parsed = [Patch.from_dict(item) for item in patches if isinstance(item, dict)]
|
||||
if len(parsed) != len(patches) or any(patch.skill_hash != skill_hash for patch in parsed):
|
||||
return None
|
||||
if [patch.role for patch in parsed] != ["promote_success", "mitigate_failure"][:len(parsed)]:
|
||||
return None
|
||||
candidate_skill = candidate / "SKILL.md"
|
||||
if not candidate_skill.is_file():
|
||||
return None
|
||||
if value.get("candidate_skill_hash") != sha256_file(candidate_skill):
|
||||
return None
|
||||
return candidate
|
||||
except (OSError, TypeError, ValueError):
|
||||
return None
|
||||
@@ -0,0 +1,97 @@
|
||||
"""动态编译唯一命令行入口。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.provider_router import parse_model_reference
|
||||
|
||||
from .pipeline import run_pipeline
|
||||
from .scoring.agentrm import (
|
||||
DEFAULT_BATCH_SIZE,
|
||||
DEFAULT_CONCURRENCY,
|
||||
DEFAULT_MAX_LENGTH,
|
||||
DEFAULT_RM_API_URL,
|
||||
DEFAULT_TIMEOUT,
|
||||
)
|
||||
|
||||
|
||||
def _provider_model(value: str) -> str:
|
||||
try:
|
||||
return parse_model_reference(value).value
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(str(exc)) from exc
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="python -m scripts.dynamic_compile.fast",
|
||||
description="从一组 BenchFlow 历史轨迹生成一个动态编译候选 Skill。",
|
||||
)
|
||||
parser.add_argument("--traces", required=True, type=Path, help="BenchFlow 轨迹目录")
|
||||
parser.add_argument("--skill", required=True, type=Path, help="含根 SKILL.md 的 Skill 包")
|
||||
parser.add_argument("--score-output", type=Path, help="评分阶段产物目录")
|
||||
parser.add_argument("--output", type=Path, help="分析与候选 Skill 产物目录")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
required=True,
|
||||
type=_provider_model,
|
||||
help="所有外部模型调用使用的 provider/model",
|
||||
)
|
||||
parser.add_argument("--max-parallel", type=int, default=3)
|
||||
parser.add_argument(
|
||||
"--rm-api-url", default=os.environ.get("RM_API_URL", DEFAULT_RM_API_URL)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rm-max-length",
|
||||
type=int,
|
||||
default=os.environ.get("RM_MAX_LENGTH", str(DEFAULT_MAX_LENGTH)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rm-timeout",
|
||||
type=float,
|
||||
default=os.environ.get("RM_TIMEOUT", str(DEFAULT_TIMEOUT)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rm-concurrency",
|
||||
type=int,
|
||||
default=os.environ.get("RM_CONCURRENCY", str(DEFAULT_CONCURRENCY)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rm-batch-size",
|
||||
type=int,
|
||||
default=os.environ.get("RM_BATCH_SIZE", str(DEFAULT_BATCH_SIZE)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="复用有效评分/Map 缓存，强制重建 Reduce、Patch 和候选 Skill",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
result = run_pipeline(
|
||||
args.traces,
|
||||
args.skill,
|
||||
score_output=args.score_output,
|
||||
output=args.output,
|
||||
model=args.model,
|
||||
max_parallel=args.max_parallel,
|
||||
rm_api_url=args.rm_api_url,
|
||||
rm_max_length=args.rm_max_length,
|
||||
rm_timeout=args.rm_timeout,
|
||||
rm_concurrency=args.rm_concurrency,
|
||||
rm_batch_size=args.rm_batch_size,
|
||||
force=args.force,
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
print(result)
|
||||
return 0
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=True)
|
||||
class TraceKey:
|
||||
task_name: str
|
||||
compile_type: str
|
||||
test_name: str
|
||||
|
||||
@classmethod
|
||||
def from_record(cls, value: Mapping[str, Any]) -> "TraceKey":
|
||||
try:
|
||||
return cls(
|
||||
str(value["task_name"]),
|
||||
str(value["compile_type"]),
|
||||
str(value["test_name"]),
|
||||
)
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"missing trace identity field: {exc.args[0]}") from exc
|
||||
|
||||
def as_tuple(self) -> tuple[str, str, str]:
|
||||
return self.task_name, self.compile_type, self.test_name
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "/".join(self.as_tuple())
|
||||
|
||||
|
||||
@dataclass
|
||||
class Patch:
|
||||
edit_type: str
|
||||
target_heading: str
|
||||
old_text: str
|
||||
new_text: str
|
||||
evidence_ids: list[str]
|
||||
evidence_type: str
|
||||
confidence: str
|
||||
reason: str
|
||||
role: str = ""
|
||||
skill_hash: str = ""
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, Any]) -> "Patch":
|
||||
required = {
|
||||
"edit_type", "target_heading", "old_text", "new_text", "evidence_ids",
|
||||
"evidence_type", "confidence", "reason",
|
||||
}
|
||||
missing = sorted(required - value.keys())
|
||||
if missing:
|
||||
raise ValueError(f"patch missing fields: {', '.join(missing)}")
|
||||
if not isinstance(value["evidence_ids"], list):
|
||||
raise ValueError("patch evidence_ids must be a list")
|
||||
if "role" in value and not isinstance(value["role"], str):
|
||||
raise ValueError("patch role must be a string")
|
||||
for key in required - {"evidence_ids"}:
|
||||
if not isinstance(value[key], str):
|
||||
raise ValueError(f"patch {key} must be a string")
|
||||
return cls(**{key: value.get(key, "") for key in cls.__dataclass_fields__})
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RolloutTrace:
|
||||
trace_id: str
|
||||
task_name: str
|
||||
compile_type: str
|
||||
test_name: str
|
||||
state: list[dict[str, Any]]
|
||||
skill_invoked: bool
|
||||
exit_code: int | None = None
|
||||
timed_out: bool | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def key(self) -> TraceKey:
|
||||
return TraceKey(self.task_name, self.compile_type, self.test_name)
|
||||
|
||||
def agentrm_request(self) -> dict[str, Any]:
|
||||
"""Project a rich runtime trace onto AgentRM's stable input schema."""
|
||||
return {
|
||||
"state": self.state,
|
||||
"task_name": self.task_name,
|
||||
"compile_type": self.compile_type,
|
||||
"test_name": self.test_name,
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..models import Patch, RolloutTrace
|
||||
from scripts.provider_router import resolve_model_route
|
||||
|
||||
from ..storage import package_manifest, sha256_file
|
||||
from .trace_format import compact_trace
|
||||
|
||||
|
||||
class SemanticClient:
|
||||
def __init__(
|
||||
self,
|
||||
model: str = "opencode/deepseek-v4-pro",
|
||||
timeout: int = 900,
|
||||
):
|
||||
route = resolve_model_route(model)
|
||||
assert route is not None
|
||||
base_url = route.url.removesuffix("/chat/completions").rstrip("/")
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("the openai package is required for semantic calls") from exc
|
||||
self.client = OpenAI(base_url=base_url, api_key=route.api_key, timeout=timeout, max_retries=0)
|
||||
self.model = route.reference.model_id
|
||||
self.model_reference = route.reference.value
|
||||
|
||||
def json(self, system: str, user: str, attempts: int = 1) -> dict[str, Any]:
|
||||
error: Exception | None = None
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
temperature=0,
|
||||
response_format={"type": "json_object"},
|
||||
messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||||
stream=True,
|
||||
)
|
||||
content = "".join(
|
||||
choice.delta.content or ""
|
||||
for chunk in response
|
||||
for choice in chunk.choices
|
||||
)
|
||||
value = json.loads(content or "{}")
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("semantic response must be a JSON object")
|
||||
return value
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
if attempt + 1 < attempts:
|
||||
print(
|
||||
f"[semantic] request attempt {attempt + 1}/{attempts} failed: "
|
||||
f"{type(exc).__name__}: {exc}; retrying",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(1 + attempt)
|
||||
raise RuntimeError(f"semantic call failed after {attempts} attempts: {error}")
|
||||
|
||||
|
||||
class SemanticAnalyzer:
|
||||
def __init__(self, client: SemanticClient, max_parallel: int = 3):
|
||||
self.client = client
|
||||
self.max_parallel = max_parallel
|
||||
|
||||
def generate_probe(self, skill_package: Path) -> dict[str, str]:
|
||||
skill = (skill_package / "SKILL.md").read_text(encoding="utf-8")
|
||||
files = [item["path"] for item in package_manifest(skill_package)]
|
||||
result = self.client.json(
|
||||
"You create realistic probe tasks for testing an agent skill. Return JSON only.",
|
||||
f"""Create one task prompt for the skill below. The task must explicitly tell the agent to invoke this skill, exercise its core workflow, and remain solvable in an empty workspace with network access. Do not copy the skill's operation steps, create a verifier, or create a test environment. Return {{"prompt": string, "rationale": string}}.
|
||||
|
||||
Package files: {json.dumps(files, ensure_ascii=False)}
|
||||
SKILL.md:
|
||||
{skill}""",
|
||||
)
|
||||
prompt = result.get("prompt")
|
||||
if not isinstance(prompt, str) or not prompt.strip():
|
||||
raise ValueError("probe generator returned no prompt")
|
||||
return {"prompt": prompt.strip(), "rationale": str(result.get("rationale", ""))}
|
||||
|
||||
def map_trace(self, trace: RolloutTrace, score: float, bucket: str) -> dict[str, Any]:
|
||||
result = self.client.json(
|
||||
"Analyze agent behavior from a scored trace. Return concise JSON only.",
|
||||
f"""Analyze this {bucket} trace (AgentRM score {score}) as an action-level workflow. Identify what the agent did, action ordering, stopping behavior, error recovery, and whether required outputs were persisted promptly. Runtime facts report observable execution only; do not infer external verification outcomes or correctness that is not visible in the trace. Use long payload details only when they are necessary to explain a behavioral effect. Return:
|
||||
{{"trace_id":"{trace.trace_id}","patterns":[{{"description":string,"condition":string,"effect":string,"recovered":boolean,"final_quality_impact":string,"evidence_ids":[string]}}]}}.
|
||||
Use E### or E###.T## event IDs as evidence. Skill invocation is evidence, not a quality gate.
|
||||
|
||||
{compact_trace(trace)}""",
|
||||
)
|
||||
result["runtime_facts"] = {
|
||||
"termination": trace.metadata.get("termination"),
|
||||
"timed_out": trace.timed_out,
|
||||
"exit_code": trace.exit_code,
|
||||
"duration_seconds": trace.metadata.get("agent_execution_seconds"),
|
||||
"tool_calls": trace.metadata.get("tool_calls"),
|
||||
"skill_invoked": trace.skill_invoked,
|
||||
"timeout_reason": trace.metadata.get("timeout_reason"),
|
||||
"error_category": trace.metadata.get("error_category"),
|
||||
"partial_trajectory": trace.metadata.get("partial_trajectory"),
|
||||
}
|
||||
result.setdefault("trace_id", trace.trace_id)
|
||||
result.setdefault("patterns", [])
|
||||
return result
|
||||
|
||||
def map_all(
|
||||
self,
|
||||
traces: list[RolloutTrace],
|
||||
scores: dict[str, float],
|
||||
high_ids: set[str],
|
||||
low_ids: set[str] | None = None,
|
||||
progress: Any | None = None,
|
||||
result_callback: Any | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
def work(trace: RolloutTrace) -> dict[str, Any]:
|
||||
bucket = "High" if trace.trace_id in high_ids else "Low" if low_ids is None or trace.trace_id in low_ids else "Neutral"
|
||||
return self.map_trace(trace, scores[trace.trace_id], bucket)
|
||||
|
||||
results: list[dict[str, Any] | None] = [None] * len(traces)
|
||||
errors: list[tuple[str, Exception]] = []
|
||||
with ThreadPoolExecutor(max_workers=self.max_parallel) as pool:
|
||||
futures = {pool.submit(work, trace): index for index, trace in enumerate(traces)}
|
||||
completed = 0
|
||||
for future in as_completed(futures):
|
||||
index = futures[future]
|
||||
try:
|
||||
result = future.result()
|
||||
except Exception as exc:
|
||||
errors.append((traces[index].trace_id, exc))
|
||||
continue
|
||||
results[index] = result
|
||||
if result_callback is not None:
|
||||
result_callback(result)
|
||||
completed += 1
|
||||
if progress is not None:
|
||||
progress(completed, len(traces), traces[index].trace_id)
|
||||
if errors:
|
||||
details = "; ".join(
|
||||
f"{trace_id}: {type(error).__name__}: {error}"
|
||||
for trace_id, error in errors
|
||||
)
|
||||
raise RuntimeError(f"{len(errors)} Map trace(s) failed; successful results were preserved: {details}")
|
||||
return [result for result in results if result is not None]
|
||||
|
||||
def reduce(
|
||||
self,
|
||||
skill_text: str,
|
||||
maps: list[dict[str, Any]],
|
||||
score_rows: list[dict[str, Any]],
|
||||
high_ids: list[str],
|
||||
low_ids: list[str],
|
||||
history: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
result = self.client.json(
|
||||
"Contrast exactly one successful pattern with exactly one failure pattern to identify one bounded skill-improvement gap. Return JSON only with every requested field populated; never return an empty object.",
|
||||
f"""Compare the fixed relative High and Low groups. Select exactly one successful behavioral pattern from the High traces that the skill should preserve or promote, and exactly one contrasting failure pattern from the Low traces that the skill should mitigate. Then select exactly one local, generalizable gap in the current skill that connects those two patterns and has not already been addressed in Patch history. Do not select two unrelated improvements.
|
||||
|
||||
The selected gap must be one behavioral clarification or stopping decision, expressed in at most two sentences, not a multi-step policy. It must explicitly preserve the selected successful behavior and mitigate the selected failure behavior, remain general to the skill, and avoid turning the failure into an exhaustive or universal obligation. Do not mention benchmark-specific files or labels, invent exact counts, thresholds, quotas, or mandatory tool sequences, add significant tool work, broaden external search, or delay a required deliverable.
|
||||
|
||||
Infer the contrast before proposing the remedy. Compare runtime_facts for completion, duration, tool calls, and timeouts; use Map patterns and their evidence to explain the behavior. Effective score indicates relative outcome, not a root cause. Describe group tendencies only to the extent supported, cite the supporting trace and event IDs, and reflect exceptions in confidence.
|
||||
|
||||
If High traces show several successful behaviors, choose the one with the clearest evidence and strongest direct contrast with the selected Low failure. If Low traces show opposing failure modes, choose only the strongest failure that can be addressed by the same qualitative decision boundary as the selected success. For incomplete work and overwork, prefer prioritization, evidence-based stopping, and timely persistence over additional checking.
|
||||
|
||||
You must still select one success, one failure, and one gap, each with a non-empty description. Each pattern must cite evidence from its corresponding group. If contrast is weak, use evidence_type=weak_contrast_fallback and confidence=low, and choose the most conservative supported pair and clarification.
|
||||
Return every field in this exact shape: {{"successful_pattern":{{"description":"one High-group behavior to preserve or promote","evidence_ids":["trace_id:E###"]}},"failure_pattern":{{"description":"one contrasting Low-group behavior to mitigate","evidence_ids":["trace_id:E###"]}},"contrast":"direct relationship between the selected success and failure","root_cause":"skill-level cause","source":"compared evidence","skill_mitigatable":true,"confidence":"low|medium|high","selected_gap":{{"description":"one supported behavioral clarification","success_behavior_to_preserve":"the selected successful behavior","failure_behavior_to_mitigate":"the selected failure behavior","target_heading":"existing skill heading","evidence_type":"contrast type","confidence":"low|medium|high","evidence_ids":["trace_id:E###"],"reason":"why this one gap preserves the success while mitigating the failure"}}}}.
|
||||
|
||||
High IDs: {json.dumps(high_ids)}
|
||||
Low IDs: {json.dumps(low_ids)}
|
||||
Scores: {json.dumps(score_rows, ensure_ascii=False)}
|
||||
Map results: {json.dumps(maps, ensure_ascii=False)}
|
||||
Patch history: {json.dumps(history, ensure_ascii=False)}
|
||||
Current SKILL.md:
|
||||
{skill_text}""",
|
||||
)
|
||||
for field, label in (
|
||||
("successful_pattern", "successful pattern"),
|
||||
("failure_pattern", "failure pattern"),
|
||||
):
|
||||
pattern = result.get(field)
|
||||
if not isinstance(pattern, dict) or not str(pattern.get("description", "")).strip():
|
||||
raise ValueError(f"reducer returned no usable {label}")
|
||||
evidence_ids = pattern.get("evidence_ids")
|
||||
if not isinstance(evidence_ids, list) or not evidence_ids:
|
||||
raise ValueError(f"reducer returned no evidence for {label}")
|
||||
gap = result.get("selected_gap")
|
||||
if not isinstance(gap, dict) or not str(gap.get("description", "")).strip():
|
||||
raise ValueError("reducer returned no usable contrastive gap")
|
||||
for field in ("success_behavior_to_preserve", "failure_behavior_to_mitigate"):
|
||||
if not str(gap.get(field, "")).strip():
|
||||
raise ValueError(f"reducer selected_gap missing {field}")
|
||||
return result
|
||||
|
||||
def generate_patches(
|
||||
self, skill_path: Path, reduction: dict[str, Any], history: list[dict[str, Any]], error: str = ""
|
||||
) -> list[Patch]:
|
||||
text = skill_path.read_text(encoding="utf-8")
|
||||
result = self.client.json(
|
||||
"Generate a small ordered patch bundle for a skill document. Return JSON only.",
|
||||
f"""Generate one required success-oriented patch and, only when it adds distinct value, one optional failure-oriented patch. Both patches must address the same selected_gap; do not introduce unrelated improvements.
|
||||
|
||||
The required promote_success patch must express the selected successful behavior as a clear, actionable recommended workflow or stopping condition in the most appropriate existing section.
|
||||
|
||||
The optional mitigate_failure patch is allowed only when it adds non-duplicative detection, recovery, or exception-handling guidance. Omit it when it would merely negate, restate, or cross-reference the promote_success patch. If included, it must remain useful independently rather than existing only to repeat the preferred path.
|
||||
|
||||
Return patches in application order: promote_success first, then optional mitigate_failure. Each patch is one contiguous text replacement. For every patch, old_text must be a non-empty, uniquely occurring verbatim substring of the original SKILL.md and patches must target non-overlapping substrings so they can be applied sequentially. new_text must replace old_text locally and preserve general applicability. Do not rewrite the whole document; keep textual growth and behavioral scope minimal.
|
||||
|
||||
The patch bundle must preserve efficient successful behavior. It must not add significant tool cost, introduce mandatory tool or API calls, broaden the existing external search scope, require exhaustive checking when targeted checking is sufficient, or delay creation of a required deliverable. Prefer prioritization, bounded stopping criteria, and writing or updating required outputs as soon as the core result is supported. Do not turn a trace-specific failure into an unconditional every/all/always/never/only-after rule unless the task itself inherently requires that rule.
|
||||
|
||||
Return {{"patches":[{{"role":"promote_success|mitigate_failure","edit_type":string,"target_heading":string,"old_text":string,"new_text":string,"evidence_ids":[string],"evidence_type":string,"confidence":string,"reason":string}}]}}. The patches array must contain one or two items and must always begin with promote_success.
|
||||
Selected analysis: {json.dumps(reduction, ensure_ascii=False)}
|
||||
History: {json.dumps(history, ensure_ascii=False)}
|
||||
Previous application error: {error}
|
||||
SKILL.md:
|
||||
{text}""",
|
||||
)
|
||||
values = result.get("patches")
|
||||
if not isinstance(values, list) or not 1 <= len(values) <= 2:
|
||||
raise ValueError("patch generator must return one or two patches")
|
||||
if not all(isinstance(value, dict) for value in values):
|
||||
raise ValueError("every generated patch must be an object")
|
||||
expected_roles = ["promote_success", "mitigate_failure"]
|
||||
roles = [value.get("role") for value in values]
|
||||
if roles != expected_roles[: len(values)]:
|
||||
raise ValueError(
|
||||
"patch roles must be promote_success followed by optional mitigate_failure"
|
||||
)
|
||||
patches = [Patch.from_dict(value) for value in values]
|
||||
if len(patches) == 2 and patches[0].new_text.strip() == patches[1].new_text.strip():
|
||||
raise ValueError("failure patch duplicates the success patch")
|
||||
skill_hash = sha256_file(skill_path)
|
||||
for patch in patches:
|
||||
patch.skill_hash = skill_hash
|
||||
return patches
|
||||
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import Patch
|
||||
from ..storage import atomic_write_text, sha256_file
|
||||
|
||||
|
||||
def _validate_skill_text(text: str) -> None:
|
||||
if not text.strip():
|
||||
raise ValueError("SKILL.md is empty")
|
||||
if text.startswith("---"):
|
||||
end = text.find("\n---", 3)
|
||||
if end < 0:
|
||||
raise ValueError("SKILL.md has an unterminated YAML frontmatter")
|
||||
frontmatter = text[3:end].strip()
|
||||
try:
|
||||
import yaml
|
||||
|
||||
parsed = yaml.safe_load(frontmatter) if frontmatter else {}
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid SKILL.md frontmatter: {exc}") from exc
|
||||
if parsed is not None and not isinstance(parsed, dict):
|
||||
raise ValueError("SKILL.md frontmatter must be a mapping")
|
||||
|
||||
|
||||
def apply_patches(
|
||||
current_package: Path, candidate_package: Path, patches: list[Patch]
|
||||
) -> str:
|
||||
skill = current_package / "SKILL.md"
|
||||
if not skill.is_file():
|
||||
raise ValueError(f"missing {skill}")
|
||||
if not patches:
|
||||
raise ValueError("at least one patch is required")
|
||||
actual_hash = sha256_file(skill)
|
||||
text = skill.read_text(encoding="utf-8")
|
||||
for index, patch in enumerate(patches, start=1):
|
||||
if not patch.skill_hash:
|
||||
raise ValueError(f"patch {index} missing generation-time skill_hash")
|
||||
if actual_hash != patch.skill_hash:
|
||||
raise ValueError(
|
||||
f"patch {index} was not generated from the current SKILL.md"
|
||||
)
|
||||
if not patch.old_text:
|
||||
raise ValueError(f"patch {index} old_text must not be empty")
|
||||
matches = text.count(patch.old_text)
|
||||
if matches != 1:
|
||||
raise ValueError(
|
||||
f"patch {index} old_text must match exactly once; found {matches}"
|
||||
)
|
||||
changed = text.replace(patch.old_text, patch.new_text, 1)
|
||||
if changed == text:
|
||||
raise ValueError(f"patch {index} does not change SKILL.md")
|
||||
_validate_skill_text(changed)
|
||||
text = changed
|
||||
token = uuid.uuid4().hex
|
||||
temporary = candidate_package.with_name(f".{candidate_package.name}.{token}.tmp")
|
||||
backup = candidate_package.with_name(f".{candidate_package.name}.{token}.bak")
|
||||
shutil.copytree(current_package, temporary)
|
||||
try:
|
||||
atomic_write_text(temporary / "SKILL.md", text)
|
||||
if candidate_package.exists():
|
||||
candidate_package.rename(backup)
|
||||
try:
|
||||
temporary.rename(candidate_package)
|
||||
except Exception:
|
||||
if backup.exists() and not candidate_package.exists():
|
||||
backup.rename(candidate_package)
|
||||
raise
|
||||
finally:
|
||||
if temporary.exists():
|
||||
shutil.rmtree(temporary)
|
||||
if backup.exists() and candidate_package.exists():
|
||||
shutil.rmtree(backup)
|
||||
return sha256_file(candidate_package / "SKILL.md")
|
||||
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
|
||||
def relative_high_low(
|
||||
score_by_id: dict[str, float], count: int = 3, seed: str | int = 0
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""稳定选择互不重叠的相对高分组和低分组。"""
|
||||
|
||||
if len(score_by_id) < count * 2:
|
||||
raise ValueError(f"need at least {count * 2} traces for disjoint High/Low groups")
|
||||
trace_ids = list(score_by_id)
|
||||
random.Random(str(seed)).shuffle(trace_ids)
|
||||
high = sorted(trace_ids, key=score_by_id.__getitem__, reverse=True)[:count]
|
||||
high_set = set(high)
|
||||
low = sorted(
|
||||
(trace_id for trace_id in trace_ids if trace_id not in high_set),
|
||||
key=score_by_id.__getitem__,
|
||||
)[:count]
|
||||
return high, low
|
||||
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from ..models import RolloutTrace
|
||||
|
||||
|
||||
_TOOL_CALL_RE = re.compile(r"^Tool call (?P<name>[^:\n]+):[ \t]*", re.MULTILINE)
|
||||
_TOOL_RESULT_RE = re.compile(
|
||||
r"^Tool result(?: \((?P<name>[^;\n)]+)(?:;[ \t]*(?P<status>[^)\n]+))?\))?:?[ \t]*",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _content(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def opencode_events_to_state(
|
||||
lines: Iterable[str], probe: str
|
||||
) -> tuple[list[dict[str, str]], bool, int]:
|
||||
state: list[dict[str, str]] = [{"role": "user", "content": probe}]
|
||||
invoked = False
|
||||
parsed = 0
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
parsed += 1
|
||||
kind = str(event.get("type", event.get("event", ""))).lower()
|
||||
part = event.get("part") if isinstance(event.get("part"), dict) else event
|
||||
tool = part.get("tool") or part.get("name") or event.get("tool") or event.get("name")
|
||||
title = part.get("title") or event.get("title") or ""
|
||||
state_data = part.get("state") if isinstance(part.get("state"), dict) else {}
|
||||
haystack = " ".join([str(kind), str(tool or ""), str(title), _content(part)])
|
||||
if str(tool or "").lower() == "skill" or "<skill_content" in haystack.lower():
|
||||
invoked = True
|
||||
if tool or "tool" in kind:
|
||||
arguments = state_data.get("input", part.get("input", part.get("arguments", {})))
|
||||
output = state_data.get("output", part.get("output", part.get("result", "")))
|
||||
state.append({"role": "assistant", "content": f"Tool call {tool or title}: {_content(arguments)}"})
|
||||
if output not in (None, ""):
|
||||
state.append({"role": "user", "content": f"Tool result: {_content(output)}"})
|
||||
continue
|
||||
text = part.get("text", part.get("content", event.get("message", "")))
|
||||
if text not in (None, ""):
|
||||
role = str(event.get("role", part.get("role", "assistant")))
|
||||
if role not in {"assistant", "user", "system"}:
|
||||
role = "assistant"
|
||||
state.append({"role": role, "content": _content(text)})
|
||||
return state, invoked, parsed
|
||||
|
||||
|
||||
def read_event_file(path: Path, probe: str) -> tuple[list[dict[str, str]], bool, int]:
|
||||
with path.open(encoding="utf-8", errors="replace") as handle:
|
||||
return opencode_events_to_state(handle, probe)
|
||||
|
||||
|
||||
def _split_tool_results(content: str) -> tuple[str, list[tuple[str, str, str]]]:
|
||||
matches = list(_TOOL_RESULT_RE.finditer(content))
|
||||
if not matches:
|
||||
return content, []
|
||||
prefix = content[:matches[0].start()].strip()
|
||||
results = []
|
||||
for index, match in enumerate(matches):
|
||||
end = matches[index + 1].start() if index + 1 < len(matches) else len(content)
|
||||
results.append((
|
||||
(match.group("name") or "unknown").strip(),
|
||||
(match.group("status") or "unknown").strip(),
|
||||
content[match.end():end].strip(),
|
||||
))
|
||||
return prefix, results
|
||||
|
||||
|
||||
def _excerpt(content: str, limit: int) -> str:
|
||||
content = content.strip()
|
||||
if len(content) <= limit:
|
||||
return content
|
||||
marker = "\n[... content omitted ...]\n"
|
||||
if limit <= len(marker) + 2:
|
||||
return content[:limit]
|
||||
omitted = len(content) - (limit - len(marker))
|
||||
while True:
|
||||
marker = f"\n[... {omitted} chars omitted ...]\n"
|
||||
available = limit - len(marker)
|
||||
updated = len(content) - available
|
||||
if updated == omitted:
|
||||
break
|
||||
omitted = updated
|
||||
head = (available + 1) // 2
|
||||
tail = available // 2
|
||||
return content[:head] + marker + content[-tail:]
|
||||
|
||||
|
||||
def _termination(trace: RolloutTrace) -> str:
|
||||
value = trace.metadata.get("termination")
|
||||
if value not in (None, ""):
|
||||
return str(value)
|
||||
if trace.timed_out is True:
|
||||
return "timeout"
|
||||
if trace.exit_code == 0:
|
||||
return "completed"
|
||||
if trace.exit_code is not None:
|
||||
return "error"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _runtime_facts(trace: RolloutTrace) -> str:
|
||||
metadata = trace.metadata
|
||||
facts = {
|
||||
"trace_id": trace.trace_id,
|
||||
"termination": _termination(trace),
|
||||
"timed_out": trace.timed_out,
|
||||
"exit_code": trace.exit_code,
|
||||
"agent_execution_seconds": metadata.get("agent_execution_seconds"),
|
||||
"tool_calls": metadata.get("tool_calls"),
|
||||
"skill_invoked": trace.skill_invoked,
|
||||
"timeout_reason": metadata.get("timeout_reason"),
|
||||
"error_category": metadata.get("error_category"),
|
||||
"partial_trajectory": metadata.get("partial_trajectory"),
|
||||
}
|
||||
return "RUNTIME_FACTS " + json.dumps(facts, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _allocate_excerpt_budget(caps: list[int], available: int) -> list[int]:
|
||||
allocations = [0] * len(caps)
|
||||
active = [index for index, cap in enumerate(caps) if cap > 0]
|
||||
while active and available > 0:
|
||||
share = max(1, available // len(active))
|
||||
progressed = False
|
||||
for index in active.copy():
|
||||
amount = min(share, caps[index] - allocations[index], available)
|
||||
allocations[index] += amount
|
||||
available -= amount
|
||||
progressed = progressed or amount > 0
|
||||
if allocations[index] >= caps[index]:
|
||||
active.remove(index)
|
||||
if available == 0:
|
||||
break
|
||||
if not progressed:
|
||||
break
|
||||
return allocations
|
||||
|
||||
|
||||
def compact_trace(trace: RolloutTrace, total: int = 30000) -> str:
|
||||
"""Render runtime facts and a complete action ledger for one Map call."""
|
||||
entries: list[dict[str, Any]] = []
|
||||
for index, message in enumerate(trace.state):
|
||||
event_id = f"E{index:03d}"
|
||||
role = str(message.get("role", "unknown"))
|
||||
content = _content(message.get("content", ""))
|
||||
|
||||
tool_call = _TOOL_CALL_RE.match(content)
|
||||
if tool_call:
|
||||
entries.append({
|
||||
"id": f"{event_id}.T01",
|
||||
"kind": "tool_call",
|
||||
"role": role,
|
||||
"name": tool_call.group("name").strip(),
|
||||
"status": "unknown",
|
||||
"content": content[tool_call.end():].strip(),
|
||||
})
|
||||
continue
|
||||
|
||||
prefix, tool_results = _split_tool_results(content) if index > 0 else (content, [])
|
||||
if prefix:
|
||||
entries.append({
|
||||
"id": event_id,
|
||||
"kind": "message",
|
||||
"role": role,
|
||||
"content": prefix,
|
||||
})
|
||||
for tool_index, (name, status, result) in enumerate(tool_results, 1):
|
||||
entries.append({
|
||||
"id": f"{event_id}.T{tool_index:02d}",
|
||||
"kind": "tool_result",
|
||||
"role": role,
|
||||
"name": name,
|
||||
"status": status,
|
||||
"content": result,
|
||||
})
|
||||
|
||||
message_entries = [entry for entry in entries if entry["kind"] == "message"]
|
||||
first_user = next((entry for entry in message_entries if entry["role"] == "user"), None)
|
||||
final_assistant = next(
|
||||
(entry for entry in reversed(message_entries) if entry["role"] == "assistant"),
|
||||
None,
|
||||
)
|
||||
|
||||
skeletons = []
|
||||
caps = []
|
||||
for entry in entries:
|
||||
if entry["kind"] == "message":
|
||||
labels = ["message", f"role={entry['role']}"]
|
||||
if entry is first_user:
|
||||
labels.append("task")
|
||||
if entry is final_assistant:
|
||||
labels.append("final")
|
||||
skeleton = f"{entry['id']} [" + " ".join(labels) + "]"
|
||||
cap = 2500 if entry is first_user else 3000 if entry is final_assistant else 600
|
||||
else:
|
||||
name = str(entry["name"])[:80]
|
||||
status = str(entry["status"])[:40]
|
||||
skeleton = f"{entry['id']} [{entry['kind']} name={name} status={status}]"
|
||||
cap = 600
|
||||
skeletons.append(skeleton)
|
||||
caps.append(min(cap, len(str(entry["content"]))))
|
||||
|
||||
facts = _runtime_facts(trace)
|
||||
fixed_size = (
|
||||
len(facts)
|
||||
+ sum(len(skeleton) + 1 for skeleton in skeletons)
|
||||
+ sum(1 for cap in caps if cap > 0)
|
||||
)
|
||||
allocations = _allocate_excerpt_budget(caps, max(0, total - fixed_size))
|
||||
rendered = [facts]
|
||||
for entry, skeleton, allocation in zip(entries, skeletons, allocations):
|
||||
rendered.append(skeleton)
|
||||
if allocation:
|
||||
rendered.append(_excerpt(str(entry["content"]), allocation))
|
||||
return "\n".join(rendered)
|
||||
@@ -0,0 +1,50 @@
|
||||
"""动态编译流水线的全部路径规则。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _find_project_root(module_dir: Path) -> Path:
|
||||
"""通过项目标志定位根目录，不编码包目录深度。"""
|
||||
|
||||
for candidate in (module_dir, *module_dir.parents):
|
||||
if (
|
||||
(candidate / "provider_routes.json").is_file()
|
||||
and (candidate / "scripts").is_dir()
|
||||
and (candidate / "data").is_dir()
|
||||
):
|
||||
return candidate
|
||||
raise RuntimeError(f"cannot locate project root from {module_dir}")
|
||||
|
||||
|
||||
PROJECT_ROOT = _find_project_root(Path(__file__).resolve().parent)
|
||||
DATA_ROOT = PROJECT_ROOT / "data"
|
||||
RESULTS_ROOT = PROJECT_ROOT / "results"
|
||||
ENV_FILE = PROJECT_ROOT / ".env"
|
||||
|
||||
DYNAMIC_RESULTS_ROOT = RESULTS_ROOT / "dynamic-optimization"
|
||||
TRACE_ROOT = DYNAMIC_RESULTS_ROOT / "traces"
|
||||
RAW_TRACE_ROOT = TRACE_ROOT / "raw_agent_trace"
|
||||
FINAL_SCORE_ROOT = TRACE_ROOT / "final-score"
|
||||
COMPILED_SKILL_ROOT = DYNAMIC_RESULTS_ROOT / "compiled-skills"
|
||||
|
||||
|
||||
def project_path(value: Path) -> Path:
|
||||
"""将命令行相对路径稳定地解释为项目根目录下的路径。"""
|
||||
|
||||
expanded = value.expanduser()
|
||||
return (expanded if expanded.is_absolute() else PROJECT_ROOT / expanded).resolve()
|
||||
|
||||
|
||||
def default_outputs(trace_input: Path, compile_type: str) -> tuple[Path, Path]:
|
||||
"""按默认原始轨迹树中的身份生成评分和候选产物目录。"""
|
||||
|
||||
try:
|
||||
relative = trace_input.resolve().relative_to(RAW_TRACE_ROOT)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
"轨迹不在默认数据树中，请同时提供 --score-output 和 --output"
|
||||
) from exc
|
||||
identity = relative.parent / compile_type
|
||||
return FINAL_SCORE_ROOT / identity, COMPILED_SKILL_ROOT / identity
|
||||
@@ -0,0 +1,319 @@
|
||||
"""动态编译主流水线；本模块只负责阶段编排。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .cache import (
|
||||
SCORE_ARTIFACTS,
|
||||
fingerprint,
|
||||
load_candidate_cache,
|
||||
load_maps_cache,
|
||||
load_reduction_cache,
|
||||
load_score_cache,
|
||||
trace_fingerprint,
|
||||
write_manifest,
|
||||
)
|
||||
from .optimization.analyzer import SemanticAnalyzer, SemanticClient
|
||||
from .optimization.patch import apply_patches
|
||||
from .optimization.selection import relative_high_low
|
||||
from .paths import default_outputs, project_path
|
||||
from .scoring.agentrm import (
|
||||
DEFAULT_BATCH_SIZE,
|
||||
DEFAULT_CONCURRENCY,
|
||||
DEFAULT_MAX_LENGTH,
|
||||
DEFAULT_RM_API_URL,
|
||||
DEFAULT_TIMEOUT,
|
||||
AgentRM,
|
||||
)
|
||||
from .scoring.service import TraceScorer
|
||||
from .scoring.pre_score import PRE_SCORE_VERSION, PreScorer, RelevanceJudge
|
||||
from .storage import (
|
||||
atomic_write_json,
|
||||
package_hash,
|
||||
read_jsonl,
|
||||
sha256_file,
|
||||
)
|
||||
from .traces.benchflow import load_benchflow_traces
|
||||
|
||||
|
||||
GROUP_SIZE = 3
|
||||
SCORE_VERSION = 1
|
||||
MAP_VERSION = 1
|
||||
REDUCTION_VERSION = 1
|
||||
PATCH_VERSION = 1
|
||||
|
||||
|
||||
def _log(message: str) -> None:
|
||||
print(f"[dynamic_compile.fast] {message}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def _implementation_name(value: object | None, default: str) -> str:
|
||||
if value is None:
|
||||
return default
|
||||
return type(value).__module__ + "." + type(value).__qualname__
|
||||
|
||||
|
||||
def run_pipeline(
|
||||
trace_input: Path,
|
||||
skill_package: Path,
|
||||
*,
|
||||
score_output: Path | None = None,
|
||||
output: Path | None = None,
|
||||
model: str = "opencode/deepseek-v4-pro",
|
||||
max_parallel: int = 3,
|
||||
rm_api_url: str | None = None,
|
||||
rm_max_length: int = DEFAULT_MAX_LENGTH,
|
||||
rm_timeout: float = DEFAULT_TIMEOUT,
|
||||
rm_concurrency: int = DEFAULT_CONCURRENCY,
|
||||
rm_batch_size: int = DEFAULT_BATCH_SIZE,
|
||||
force: bool = False,
|
||||
analyzer: SemanticAnalyzer | None = None,
|
||||
pre_scorer: PreScorer | None = None,
|
||||
agentrm: AgentRM | None = None,
|
||||
) -> Path:
|
||||
"""从历史 BenchFlow 轨迹生成一个候选 Skill 包。"""
|
||||
|
||||
trace_input = project_path(trace_input)
|
||||
skill_package = project_path(skill_package)
|
||||
if not (skill_package / "SKILL.md").is_file():
|
||||
raise ValueError("skill package must contain a root SKILL.md")
|
||||
if max_parallel < 1:
|
||||
raise ValueError("max_parallel must be at least 1")
|
||||
if agentrm is None and min(
|
||||
rm_max_length, rm_timeout, rm_concurrency, rm_batch_size
|
||||
) <= 0:
|
||||
raise ValueError("AgentRM numeric options must be positive")
|
||||
|
||||
traces = load_benchflow_traces(trace_input)
|
||||
identities = {(trace.task_name, trace.compile_type) for trace in traces}
|
||||
if len(identities) != 1:
|
||||
raise ValueError(
|
||||
f"trace input must contain one task and compile type: {sorted(identities)}"
|
||||
)
|
||||
if len(traces) < GROUP_SIZE * 2:
|
||||
raise ValueError(f"at least {GROUP_SIZE * 2} traces are required")
|
||||
if len({trace.test_name for trace in traces}) != len(traces):
|
||||
raise ValueError("trace input contains duplicate test names")
|
||||
|
||||
_, compile_type = next(iter(identities))
|
||||
if score_output is None or output is None:
|
||||
default_score, default_output = default_outputs(trace_input, compile_type)
|
||||
score_dir = project_path(score_output) if score_output else default_score
|
||||
output_dir = project_path(output) if output else default_output
|
||||
if output_dir == skill_package or output_dir.is_relative_to(skill_package):
|
||||
raise ValueError("output directory must not be inside the input skill package")
|
||||
score_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
traces_hash = trace_fingerprint(traces)
|
||||
resolved_rm_url = rm_api_url or os.environ.get("RM_API_URL", DEFAULT_RM_API_URL)
|
||||
score_config: dict[str, Any] = {
|
||||
"score_version": SCORE_VERSION,
|
||||
"pre_score_version": PRE_SCORE_VERSION,
|
||||
"model": model,
|
||||
"pre_scorer": _implementation_name(pre_scorer, "PreScorer/RelevanceJudge"),
|
||||
"agentrm": _implementation_name(agentrm, "AgentRM/HttpAgentRMBackend"),
|
||||
"rm_api_url": resolved_rm_url,
|
||||
"rm_max_length": rm_max_length,
|
||||
"rm_timeout": rm_timeout,
|
||||
"rm_concurrency": rm_concurrency,
|
||||
"rm_batch_size": rm_batch_size,
|
||||
}
|
||||
score_input_hash = fingerprint({"traces": traces_hash, "config": score_config})
|
||||
|
||||
_log(f"loaded {len(traces)} traces from {trace_input}")
|
||||
cached_score = load_score_cache(
|
||||
traces, score_dir, input_hash=score_input_hash, config=score_config
|
||||
)
|
||||
if cached_score is not None:
|
||||
scores, score_rows = cached_score
|
||||
_log("reusing complete, input-matched scoring cache")
|
||||
else:
|
||||
scoring = TraceScorer(
|
||||
pre_scorer or PreScorer(RelevanceJudge(model=model), max_parallel),
|
||||
agentrm
|
||||
or AgentRM(
|
||||
api_url=resolved_rm_url,
|
||||
max_length=rm_max_length,
|
||||
timeout=rm_timeout,
|
||||
concurrency=rm_concurrency,
|
||||
batch_size=rm_batch_size,
|
||||
),
|
||||
)
|
||||
_log("scoring traces")
|
||||
scores = scoring.score_all(traces, score_dir)
|
||||
score_rows = read_jsonl(score_dir / "effective_scores.jsonl")
|
||||
write_manifest(
|
||||
score_dir,
|
||||
".score-cache.json",
|
||||
stage="score",
|
||||
input_hash=score_input_hash,
|
||||
config=score_config,
|
||||
artifacts=SCORE_ARTIFACTS,
|
||||
)
|
||||
|
||||
high, low = relative_high_low(scores, count=GROUP_SIZE)
|
||||
selected = high + low
|
||||
semantic: SemanticAnalyzer | None = analyzer
|
||||
|
||||
def get_semantic() -> SemanticAnalyzer:
|
||||
nonlocal semantic
|
||||
if semantic is None:
|
||||
semantic = SemanticAnalyzer(SemanticClient(model), max_parallel)
|
||||
return semantic
|
||||
|
||||
semantic_implementation = _implementation_name(analyzer, "SemanticAnalyzer/SemanticClient")
|
||||
map_config = {
|
||||
"map_version": MAP_VERSION,
|
||||
"model": model,
|
||||
"semantic_implementation": semantic_implementation,
|
||||
"max_parallel": max_parallel,
|
||||
}
|
||||
map_input_hash = fingerprint(
|
||||
{
|
||||
"traces": traces_hash,
|
||||
"selected": selected,
|
||||
"scores": {trace_id: scores[trace_id] for trace_id in selected},
|
||||
}
|
||||
)
|
||||
maps = load_maps_cache(
|
||||
output_dir,
|
||||
selected,
|
||||
input_hash=map_input_hash,
|
||||
config=map_config,
|
||||
)
|
||||
if maps is not None:
|
||||
_log(f"reusing complete Top {GROUP_SIZE} / Bottom {GROUP_SIZE} Map cache")
|
||||
else:
|
||||
_log(f"mapping Top {GROUP_SIZE} / Bottom {GROUP_SIZE} traces")
|
||||
selected_set = set(selected)
|
||||
maps = get_semantic().map_all(
|
||||
[trace for trace in traces if trace.trace_id in selected_set],
|
||||
scores,
|
||||
set(high),
|
||||
set(low),
|
||||
progress=lambda done, total, trace_id: _log(
|
||||
f"Map {done}/{total}: {trace_id}"
|
||||
),
|
||||
)
|
||||
atomic_write_json(output_dir / "maps.json", maps)
|
||||
write_manifest(
|
||||
output_dir,
|
||||
".maps-cache.json",
|
||||
stage="maps",
|
||||
input_hash=map_input_hash,
|
||||
config=map_config,
|
||||
artifacts=("maps.json",),
|
||||
)
|
||||
|
||||
maps_by_id = {str(item["trace_id"]): item for item in maps}
|
||||
scores_by_id = {str(item["trace_id"]): item for item in score_rows}
|
||||
skill_path = skill_package / "SKILL.md"
|
||||
skill_text = skill_path.read_text(encoding="utf-8")
|
||||
skill_hash = sha256_file(skill_path)
|
||||
skill_package_hash = package_hash(skill_package)
|
||||
reduction_config = {
|
||||
"reduction_version": REDUCTION_VERSION,
|
||||
"model": model,
|
||||
"semantic_implementation": semantic_implementation,
|
||||
}
|
||||
reduction_input_hash = fingerprint(
|
||||
{
|
||||
"traces": traces_hash,
|
||||
"package": skill_package_hash,
|
||||
"maps": [maps_by_id[trace_id] for trace_id in selected],
|
||||
"score_rows": [scores_by_id[trace_id] for trace_id in selected],
|
||||
"high": high,
|
||||
"low": low,
|
||||
}
|
||||
)
|
||||
reduction = None if force else load_reduction_cache(
|
||||
output_dir,
|
||||
input_hash=reduction_input_hash,
|
||||
config=reduction_config,
|
||||
)
|
||||
if reduction is not None:
|
||||
_log("reusing complete Top/Bottom reduction cache")
|
||||
else:
|
||||
_log("reducing Top/Bottom contrast")
|
||||
reduction = get_semantic().reduce(
|
||||
skill_text,
|
||||
[maps_by_id[trace_id] for trace_id in selected],
|
||||
[scores_by_id[trace_id] for trace_id in selected],
|
||||
high,
|
||||
low,
|
||||
[],
|
||||
)
|
||||
atomic_write_json(output_dir / "reduction.json", reduction)
|
||||
write_manifest(
|
||||
output_dir,
|
||||
".reduction-cache.json",
|
||||
stage="reduction",
|
||||
input_hash=reduction_input_hash,
|
||||
config=reduction_config,
|
||||
artifacts=("reduction.json",),
|
||||
)
|
||||
|
||||
candidate = output_dir / "candidate-skill" / skill_package.name
|
||||
candidate_config = {
|
||||
"patch_version": PATCH_VERSION,
|
||||
"model": model,
|
||||
"semantic_implementation": semantic_implementation,
|
||||
"attempts": 3,
|
||||
}
|
||||
candidate_input_hash = fingerprint(
|
||||
{
|
||||
"traces": traces_hash,
|
||||
"package": skill_package_hash,
|
||||
"reduction": reduction,
|
||||
}
|
||||
)
|
||||
cached_candidate = None if force else load_candidate_cache(
|
||||
output_dir,
|
||||
skill_package.name,
|
||||
skill_hash,
|
||||
input_hash=candidate_input_hash,
|
||||
config=candidate_config,
|
||||
)
|
||||
if cached_candidate is not None:
|
||||
_log(f"reusing complete candidate skill: {cached_candidate}")
|
||||
return cached_candidate
|
||||
|
||||
error = ""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
_log(f"generating patch bundle ({attempt + 1}/3)")
|
||||
patches = get_semantic().generate_patches(skill_path, reduction, [], error)
|
||||
candidate_hash = apply_patches(skill_package, candidate, patches)
|
||||
atomic_write_json(
|
||||
output_dir / "patch.json",
|
||||
{
|
||||
"patches": [patch.to_dict() for patch in patches],
|
||||
"skill_hash": skill_hash,
|
||||
"candidate_skill_hash": candidate_hash,
|
||||
},
|
||||
)
|
||||
write_manifest(
|
||||
output_dir,
|
||||
".candidate-cache.json",
|
||||
stage="candidate",
|
||||
input_hash=candidate_input_hash,
|
||||
config=candidate_config,
|
||||
artifacts=(
|
||||
"patch.json",
|
||||
f"candidate-skill/{skill_package.name}",
|
||||
),
|
||||
)
|
||||
_log(f"candidate skill ready: {candidate}")
|
||||
return candidate
|
||||
except (OSError, ValueError, RuntimeError) as exc:
|
||||
error = str(exc)
|
||||
if attempt == 2:
|
||||
raise RuntimeError(
|
||||
f"could not generate an applicable patch: {error}"
|
||||
) from exc
|
||||
raise AssertionError("unreachable")
|
||||
@@ -0,0 +1,191 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Protocol
|
||||
|
||||
import requests
|
||||
|
||||
from .identity import composite_id
|
||||
|
||||
|
||||
DEFAULT_MAX_LENGTH = 8192
|
||||
DEFAULT_RM_API_URL = "http://127.0.0.1:28080"
|
||||
DEFAULT_TIMEOUT = 300.0
|
||||
DEFAULT_CONCURRENCY = 8
|
||||
DEFAULT_BATCH_SIZE = 32
|
||||
RETRY_ATTEMPTS = 3
|
||||
|
||||
|
||||
class AgentRMBackend(Protocol):
|
||||
def score(self, requests: list[dict[str, Any]]) -> list[dict[str, Any]]: ...
|
||||
|
||||
|
||||
class HttpAgentRMBackend:
|
||||
"""AgentRM backend backed by the remote ``/score_batch`` API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_url: str | None = None,
|
||||
*,
|
||||
max_length: int = DEFAULT_MAX_LENGTH,
|
||||
timeout: float = DEFAULT_TIMEOUT,
|
||||
concurrency: int = DEFAULT_CONCURRENCY,
|
||||
batch_size: int = DEFAULT_BATCH_SIZE,
|
||||
) -> None:
|
||||
self.api_url = (
|
||||
api_url or os.environ.get("RM_API_URL", DEFAULT_RM_API_URL)
|
||||
).rstrip("/")
|
||||
self.max_length = max_length
|
||||
self.timeout = timeout
|
||||
self.concurrency = concurrency
|
||||
self.batch_size = batch_size
|
||||
if not self.api_url:
|
||||
raise ValueError("AgentRM API URL cannot be empty")
|
||||
if max_length <= 0 or timeout <= 0 or concurrency <= 0 or batch_size <= 0:
|
||||
raise ValueError(
|
||||
"AgentRM max_length, timeout, concurrency, and batch_size must be positive"
|
||||
)
|
||||
|
||||
def _post_batch(self, batch: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
payload = {
|
||||
"states": [request["state"] for request in batch],
|
||||
"max_length": self.max_length,
|
||||
}
|
||||
last_error: BaseException | None = None
|
||||
for attempt in range(RETRY_ATTEMPTS):
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{self.api_url}/score_batch",
|
||||
json=payload,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
scores = body.get("scores") if isinstance(body, dict) else None
|
||||
if not isinstance(scores, list) or len(scores) != len(batch):
|
||||
count = len(scores) if isinstance(scores, list) else "invalid"
|
||||
raise ValueError(
|
||||
f"AgentRM returned {count} scores for {len(batch)} states"
|
||||
)
|
||||
if any(not isinstance(score, dict) for score in scores):
|
||||
raise ValueError("AgentRM returned a non-object score item")
|
||||
if any(
|
||||
"score" not in score or "n_tokens" not in score
|
||||
for score in scores
|
||||
):
|
||||
raise ValueError("AgentRM returned a score item with missing fields")
|
||||
return [
|
||||
{
|
||||
"task_name": request["task_name"],
|
||||
"compile_type": request["compile_type"],
|
||||
"test_name": request["test_name"],
|
||||
**score,
|
||||
}
|
||||
for request, score in zip(batch, scores)
|
||||
]
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
last_error = exc
|
||||
if attempt + 1 < RETRY_ATTEMPTS:
|
||||
time.sleep(2**attempt)
|
||||
assert last_error is not None
|
||||
raise RuntimeError(
|
||||
f"AgentRM request failed after {RETRY_ATTEMPTS} attempts: {last_error}"
|
||||
)
|
||||
|
||||
def score(self, requests_to_score: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
if not requests_to_score:
|
||||
return []
|
||||
batches = [
|
||||
requests_to_score[index : index + self.batch_size]
|
||||
for index in range(0, len(requests_to_score), self.batch_size)
|
||||
]
|
||||
ordered: list[list[dict[str, Any]] | None] = [None] * len(batches)
|
||||
with ThreadPoolExecutor(max_workers=self.concurrency) as pool:
|
||||
futures = {
|
||||
pool.submit(self._post_batch, batch): index
|
||||
for index, batch in enumerate(batches)
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
ordered[futures[future]] = future.result()
|
||||
return [row for batch in ordered if batch is not None for row in batch]
|
||||
|
||||
|
||||
class AgentRM:
|
||||
"""Score AgentRM requests through a validated, replaceable backend."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
backend: AgentRMBackend | None = None,
|
||||
*,
|
||||
api_url: str | None = None,
|
||||
max_length: int = DEFAULT_MAX_LENGTH,
|
||||
timeout: float = DEFAULT_TIMEOUT,
|
||||
concurrency: int = DEFAULT_CONCURRENCY,
|
||||
batch_size: int = DEFAULT_BATCH_SIZE,
|
||||
) -> None:
|
||||
self.backend = (
|
||||
backend
|
||||
if backend is not None
|
||||
else HttpAgentRMBackend(
|
||||
api_url,
|
||||
max_length=max_length,
|
||||
timeout=timeout,
|
||||
concurrency=concurrency,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
)
|
||||
|
||||
def score_requests(
|
||||
self, requests_to_score: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
request_keys = [composite_id(request) for request in requests_to_score]
|
||||
if len(set(request_keys)) != len(request_keys):
|
||||
raise ValueError("AgentRM requests contain duplicate identities")
|
||||
|
||||
responses = self.backend.score(requests_to_score)
|
||||
response_by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
for response in responses:
|
||||
key = composite_id(response)
|
||||
if key in response_by_key:
|
||||
raise ValueError(f"AgentRM returned duplicate score identity: {key}")
|
||||
response_by_key[key] = response
|
||||
|
||||
requested = set(request_keys)
|
||||
unexpected = set(response_by_key) - requested
|
||||
missing = requested - set(response_by_key)
|
||||
if unexpected:
|
||||
raise ValueError(
|
||||
f"AgentRM returned unexpected score identities: {sorted(unexpected)}"
|
||||
)
|
||||
if missing:
|
||||
raise ValueError(f"AgentRM returned incomplete scores: {sorted(missing)}")
|
||||
|
||||
result = []
|
||||
for request, key in zip(requests_to_score, request_keys):
|
||||
response = response_by_key[key]
|
||||
try:
|
||||
score = float(response["score"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise ValueError(f"AgentRM returned an invalid score for {key}") from exc
|
||||
if not math.isfinite(score):
|
||||
raise ValueError(f"AgentRM returned a non-finite score for {key}")
|
||||
try:
|
||||
n_tokens = int(response["n_tokens"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise ValueError(
|
||||
f"AgentRM returned an invalid n_tokens for {key}"
|
||||
) from exc
|
||||
if n_tokens < 0:
|
||||
raise ValueError(f"AgentRM returned a negative n_tokens for {key}")
|
||||
row = {
|
||||
"task_name": str(request["task_name"]),
|
||||
"compile_type": str(request["compile_type"]),
|
||||
"test_name": str(request["test_name"]),
|
||||
"score": score,
|
||||
"n_tokens": n_tokens,
|
||||
}
|
||||
result.append(row)
|
||||
return result
|
||||
@@ -0,0 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..models import TraceKey
|
||||
|
||||
|
||||
ScoreKey = tuple[str, str, str]
|
||||
|
||||
|
||||
def composite_id(row: dict[str, Any]) -> ScoreKey:
|
||||
return TraceKey.from_record(row).as_tuple()
|
||||
return result
|
||||
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from typing import Any, Protocol
|
||||
|
||||
from ..models import RolloutTrace
|
||||
from ..optimization.analyzer import SemanticClient
|
||||
|
||||
|
||||
RELEVANCE_MODEL = "opencode/deepseek-v4-pro"
|
||||
RELEVANCE_CONFIDENCE_THRESHOLD = 0.80
|
||||
TIMEOUT_SCORE = 0.0
|
||||
IRRELEVANT_SCORE = 0.1
|
||||
MAX_EFFICIENCY_PENALTY = 0.08
|
||||
TIME_COST_WEIGHT = 0.80
|
||||
TOOL_COST_WEIGHT = 0.20
|
||||
EFFICIENCY_WINSOR_QUANTILE = 0.90
|
||||
PRE_SCORE_VERSION = 3
|
||||
|
||||
|
||||
class RelevanceClient(Protocol):
|
||||
def json(self, system: str, user: str, attempts: int = 3) -> dict[str, Any]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreScoreResult:
|
||||
trace_id: str
|
||||
route: str
|
||||
score: float | None
|
||||
reason: str
|
||||
relevance_label: str | None = None
|
||||
relevance_confidence: float | None = None
|
||||
efficiency_cost: float = 0.0
|
||||
efficiency_penalty: float = 0.0
|
||||
scoring_version: int = PRE_SCORE_VERSION
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, Any]) -> "PreScoreResult":
|
||||
return cls(
|
||||
trace_id=str(value["trace_id"]),
|
||||
route=str(value["route"]),
|
||||
score=float(value["score"]) if value.get("score") is not None else None,
|
||||
reason=str(value["reason"]),
|
||||
relevance_label=(
|
||||
str(value["relevance_label"])
|
||||
if value.get("relevance_label") is not None
|
||||
else None
|
||||
),
|
||||
relevance_confidence=(
|
||||
float(value["relevance_confidence"])
|
||||
if value.get("relevance_confidence") is not None
|
||||
else None
|
||||
),
|
||||
efficiency_cost=float(value.get("efficiency_cost", 0.0)),
|
||||
efficiency_penalty=float(value.get("efficiency_penalty", 0.0)),
|
||||
scoring_version=int(value.get("scoring_version", 1)),
|
||||
)
|
||||
|
||||
|
||||
def adjust_agent_score(score: float, pre_score: PreScoreResult) -> float:
|
||||
"""Apply the batch-relative efficiency penalty to an AgentRM quality score."""
|
||||
if pre_score.route != "agentrm":
|
||||
return score
|
||||
return max(IRRELEVANT_SCORE, score - pre_score.efficiency_penalty)
|
||||
|
||||
|
||||
def _quantile(values: list[float], quantile: float) -> float:
|
||||
ordered = sorted(values)
|
||||
if len(ordered) == 1:
|
||||
return ordered[0]
|
||||
position = (len(ordered) - 1) * quantile
|
||||
lower = math.floor(position)
|
||||
upper = math.ceil(position)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
fraction = position - lower
|
||||
return ordered[lower] + fraction * (ordered[upper] - ordered[lower])
|
||||
|
||||
|
||||
def _magnitude_costs(
|
||||
values: dict[str, float],
|
||||
*,
|
||||
transform=lambda value: value,
|
||||
power: float = 1.0,
|
||||
) -> dict[str, float]:
|
||||
if len(values) < 2:
|
||||
return {trace_id: 0.0 for trace_id in values}
|
||||
transformed = {trace_id: float(transform(value)) for trace_id, value in values.items()}
|
||||
floor = min(transformed.values())
|
||||
ceiling = _quantile(list(transformed.values()), EFFICIENCY_WINSOR_QUANTILE)
|
||||
if ceiling <= floor:
|
||||
return {trace_id: 0.0 for trace_id in values}
|
||||
scale = ceiling - floor
|
||||
return {
|
||||
trace_id: min(1.0, max(0.0, (value - floor) / scale)) ** power
|
||||
for trace_id, value in transformed.items()
|
||||
}
|
||||
|
||||
|
||||
class RelevanceJudge:
|
||||
def __init__(
|
||||
self,
|
||||
client: RelevanceClient | None = None,
|
||||
model: str = RELEVANCE_MODEL,
|
||||
):
|
||||
self.model = model
|
||||
self.client = client or SemanticClient(model)
|
||||
|
||||
def judge(self, task_prompt: str, final_output: str) -> tuple[str, float]:
|
||||
result = self.client.json(
|
||||
"Judge only whether an agent's last output is relevant to its task. Return JSON only.",
|
||||
f"""Classify the last agent output as relevant or irrelevant to the task.
|
||||
|
||||
An output is relevant if it attempts, plans, discusses, or reports work on the requested task, even when it is wrong, incomplete, brief, malformed, or lacks a final answer. Mark it irrelevant only when it clearly addresses a materially different task or topic. Do not judge correctness, completeness, or answer quality.
|
||||
|
||||
Return exactly {{"label":"relevant"|"irrelevant","confidence":number}} where confidence is between 0 and 1.
|
||||
|
||||
Input:
|
||||
{json.dumps({"task_prompt": task_prompt, "last_agent_output": final_output}, ensure_ascii=False)}""",
|
||||
)
|
||||
if not isinstance(result, dict) or set(result) != {"label", "confidence"}:
|
||||
raise ValueError("relevance response must contain exactly label and confidence")
|
||||
label = result.get("label")
|
||||
confidence = result.get("confidence")
|
||||
if label not in {"relevant", "irrelevant"}:
|
||||
raise ValueError("relevance label must be relevant or irrelevant")
|
||||
if isinstance(confidence, bool) or not isinstance(confidence, (int, float)):
|
||||
raise ValueError("relevance confidence must be a number")
|
||||
confidence = float(confidence)
|
||||
if not math.isfinite(confidence) or not 0.0 <= confidence <= 1.0:
|
||||
raise ValueError("relevance confidence must be between 0 and 1")
|
||||
return label, confidence
|
||||
|
||||
|
||||
class PreScorer:
|
||||
def __init__(
|
||||
self,
|
||||
judge: RelevanceJudge,
|
||||
max_parallel: int = 3,
|
||||
confidence_threshold: float = RELEVANCE_CONFIDENCE_THRESHOLD,
|
||||
):
|
||||
self.judge = judge
|
||||
self.max_parallel = max(1, max_parallel)
|
||||
self.confidence_threshold = confidence_threshold
|
||||
|
||||
@staticmethod
|
||||
def _first_user_message(trace: RolloutTrace) -> str:
|
||||
return next(
|
||||
(
|
||||
str(message.get("content", "")).strip()
|
||||
for message in trace.state
|
||||
if isinstance(message, dict)
|
||||
and message.get("role") == "user"
|
||||
and str(message.get("content", "")).strip()
|
||||
),
|
||||
"",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _last_assistant_message(trace: RolloutTrace) -> str:
|
||||
return next(
|
||||
(
|
||||
str(message.get("content", "")).strip()
|
||||
for message in reversed(trace.state)
|
||||
if isinstance(message, dict)
|
||||
and message.get("role") == "assistant"
|
||||
and str(message.get("content", "")).strip()
|
||||
),
|
||||
"",
|
||||
)
|
||||
|
||||
def score_one(self, trace: RolloutTrace) -> PreScoreResult:
|
||||
if trace.timed_out:
|
||||
return PreScoreResult(trace.trace_id, "fixed_score", TIMEOUT_SCORE, "hard_timeout")
|
||||
|
||||
final_output = self._last_assistant_message(trace)
|
||||
if not final_output:
|
||||
return PreScoreResult(trace.trace_id, "agentrm", None, "no_assistant_output")
|
||||
|
||||
task_prompt = self._first_user_message(trace)
|
||||
try:
|
||||
label, confidence = self.judge.judge(task_prompt, final_output)
|
||||
except Exception:
|
||||
return PreScoreResult(trace.trace_id, "agentrm", None, "judge_failed")
|
||||
|
||||
if label == "irrelevant" and confidence >= self.confidence_threshold:
|
||||
return PreScoreResult(
|
||||
trace.trace_id,
|
||||
"fixed_score",
|
||||
IRRELEVANT_SCORE,
|
||||
"strongly_irrelevant",
|
||||
label,
|
||||
confidence,
|
||||
)
|
||||
reason = "low_confidence" if label == "irrelevant" else "relevant"
|
||||
return PreScoreResult(
|
||||
trace.trace_id, "agentrm", None, reason, label, confidence
|
||||
)
|
||||
|
||||
def score_all(self, traces: list[RolloutTrace]) -> list[PreScoreResult]:
|
||||
results: list[PreScoreResult | None] = [None] * len(traces)
|
||||
with ThreadPoolExecutor(max_workers=self.max_parallel) as pool:
|
||||
futures = {
|
||||
pool.submit(self.score_one, trace): index
|
||||
for index, trace in enumerate(traces)
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
results[futures[future]] = future.result()
|
||||
completed = [result for result in results if result is not None]
|
||||
by_id = {trace.trace_id: trace for trace in traces}
|
||||
eligible = {
|
||||
result.trace_id for result in completed if result.route == "agentrm"
|
||||
}
|
||||
durations = {
|
||||
trace_id: float(by_id[trace_id].metadata["agent_execution_seconds"])
|
||||
for trace_id in eligible
|
||||
if isinstance(
|
||||
by_id[trace_id].metadata.get("agent_execution_seconds"), (int, float)
|
||||
)
|
||||
}
|
||||
tool_calls = {
|
||||
trace_id: float(by_id[trace_id].metadata["tool_calls"])
|
||||
for trace_id in eligible
|
||||
if isinstance(by_id[trace_id].metadata.get("tool_calls"), (int, float))
|
||||
}
|
||||
duration_cost = _magnitude_costs(durations, power=2.0)
|
||||
tool_cost = _magnitude_costs(tool_calls, transform=math.log1p)
|
||||
adjusted = []
|
||||
for result in completed:
|
||||
components = []
|
||||
if result.trace_id in duration_cost:
|
||||
components.append((TIME_COST_WEIGHT, duration_cost[result.trace_id]))
|
||||
if result.trace_id in tool_cost:
|
||||
components.append((TOOL_COST_WEIGHT, tool_cost[result.trace_id]))
|
||||
total_weight = sum(weight for weight, _ in components)
|
||||
cost = (
|
||||
sum(weight * value for weight, value in components) / total_weight
|
||||
if total_weight
|
||||
else 0.0
|
||||
)
|
||||
adjusted.append(
|
||||
replace(
|
||||
result,
|
||||
efficiency_cost=cost,
|
||||
efficiency_penalty=MAX_EFFICIENCY_PENALTY * cost,
|
||||
)
|
||||
)
|
||||
return adjusted
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import RolloutTrace
|
||||
from ..storage import atomic_write_jsonl
|
||||
from .agentrm import AgentRM
|
||||
from .pre_score import PreScorer, adjust_agent_score
|
||||
from .identity import composite_id
|
||||
|
||||
|
||||
class TraceScorer:
|
||||
"""Resolve pre-scores and AgentRM scores into one effective score per trace."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pre_scorer: PreScorer,
|
||||
agentrm: AgentRM | None = None,
|
||||
):
|
||||
self.pre_scorer = pre_scorer
|
||||
self.agentrm = agentrm if agentrm is not None else AgentRM()
|
||||
|
||||
def score_all(
|
||||
self,
|
||||
traces: list[RolloutTrace],
|
||||
final_score_dir: Path,
|
||||
agentrm_dir: Path | None = None,
|
||||
) -> dict[str, float]:
|
||||
final_score_dir.mkdir(parents=True, exist_ok=True)
|
||||
agentrm_dir = agentrm_dir or final_score_dir
|
||||
agentrm_dir.mkdir(parents=True, exist_ok=True)
|
||||
results = self.pre_scorer.score_all(traces)
|
||||
pre_scores = {result.trace_id: result for result in results}
|
||||
if set(pre_scores) != {trace.trace_id for trace in traces}:
|
||||
raise ValueError("pre-scorer returned incomplete or duplicate results")
|
||||
atomic_write_jsonl(
|
||||
final_score_dir / "pre_scores.jsonl",
|
||||
[pre_scores[trace.trace_id].to_dict() for trace in traces],
|
||||
)
|
||||
agentrm_traces = [
|
||||
trace for trace in traces if pre_scores[trace.trace_id].route == "agentrm"
|
||||
]
|
||||
requests = [trace.agentrm_request() for trace in agentrm_traces]
|
||||
local_scores = agentrm_dir / "agentrm_scores.jsonl"
|
||||
found = self.agentrm.score_requests(requests)
|
||||
atomic_write_jsonl(local_scores, found)
|
||||
|
||||
by_key = {composite_id(row): float(row["score"]) for row in found}
|
||||
resolved = []
|
||||
scores: dict[str, float] = {}
|
||||
for trace in traces:
|
||||
pre_score = pre_scores[trace.trace_id]
|
||||
if pre_score.route == "fixed_score":
|
||||
if pre_score.score is None:
|
||||
raise ValueError(f"fixed pre-score is missing for {trace.trace_id}")
|
||||
score = pre_score.score
|
||||
source = "pre_score"
|
||||
else:
|
||||
score = adjust_agent_score(by_key[trace.key.as_tuple()], pre_score)
|
||||
source = "agentrm"
|
||||
scores[trace.trace_id] = score
|
||||
resolved.append({
|
||||
"task_name": trace.task_name,
|
||||
"compile_type": trace.compile_type,
|
||||
"test_name": trace.test_name,
|
||||
"trace_id": trace.trace_id,
|
||||
"effective_score": score,
|
||||
"score_source": source,
|
||||
"pre_score_reason": pre_score.reason,
|
||||
"agentrm_score": by_key.get(trace.key.as_tuple()),
|
||||
"efficiency_cost": pre_score.efficiency_cost,
|
||||
"efficiency_penalty": pre_score.efficiency_penalty,
|
||||
})
|
||||
atomic_write_jsonl(final_score_dir / "effective_scores.jsonl", resolved)
|
||||
return scores
|
||||
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
MAX_CHARS = 2000
|
||||
LAST_MSG_BUDGET = 4000
|
||||
HEAD_RATIO = 0.5
|
||||
|
||||
|
||||
def compact(content: str, budget: int) -> str:
|
||||
if len(content) <= budget:
|
||||
return content
|
||||
head = int(budget * HEAD_RATIO)
|
||||
tail = budget - head - 50
|
||||
return (
|
||||
content[:head]
|
||||
+ f"\n...[truncated {len(content)-head-tail} chars]...\n"
|
||||
+ content[-tail:]
|
||||
)
|
||||
|
||||
|
||||
def _unwrap_text_content(value: str) -> str:
|
||||
prefix = "@{type=text; text="
|
||||
if value.startswith(prefix) and value.endswith("}"):
|
||||
return value[len(prefix) : -1]
|
||||
return value
|
||||
|
||||
|
||||
def _tool_result_text(event: dict[str, Any]) -> str:
|
||||
parts: list[str] = []
|
||||
content_items = event.get("content")
|
||||
if isinstance(content_items, list):
|
||||
for item in content_items:
|
||||
if not isinstance(item, dict) or item.get("type") != "content":
|
||||
continue
|
||||
value = item.get("content")
|
||||
if isinstance(value, str) and value:
|
||||
parts.append(_unwrap_text_content(value))
|
||||
elif (
|
||||
isinstance(value, dict)
|
||||
and value.get("type") == "text"
|
||||
and isinstance(value.get("text"), str)
|
||||
and value["text"]
|
||||
):
|
||||
parts.append(value["text"])
|
||||
return "\n".join(parts) if parts else "(no output)"
|
||||
|
||||
|
||||
def acp_events_to_state(events: Iterable[dict[str, Any]]) -> list[dict[str, str]]:
|
||||
merged: list[dict[str, str]] = []
|
||||
for event in events:
|
||||
event_type = event.get("type")
|
||||
role: str | None = None
|
||||
content: str | None = None
|
||||
if event_type == "user_message":
|
||||
role, content = "user", event.get("text")
|
||||
elif event_type == "agent_thought":
|
||||
role, content = "assistant", event.get("text")
|
||||
elif event_type == "agent_message":
|
||||
role, content = "assistant", event.get("text")
|
||||
if content == "":
|
||||
continue
|
||||
elif event_type == "tool_call":
|
||||
role = "user"
|
||||
kind = event.get("kind", "other")
|
||||
status = event.get("status", "unknown")
|
||||
content = f"Tool result ({kind}; {status}):\n{_tool_result_text(event)}"
|
||||
elif event_type == "agent_timeout":
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
||||
if not isinstance(content, str):
|
||||
raise ValueError(f"{event_type} event has a non-string text/content value")
|
||||
if merged and merged[-1]["role"] == role:
|
||||
merged[-1]["content"] += "\n" + content
|
||||
else:
|
||||
merged.append({"role": role, "content": content})
|
||||
|
||||
if not merged:
|
||||
raise ValueError("trajectory produced an empty state")
|
||||
if merged[0]["role"] != "user":
|
||||
raise ValueError("state must start with a user message")
|
||||
|
||||
seen: set[str] = set()
|
||||
last_index = len(merged) - 1
|
||||
for index, message in enumerate(merged):
|
||||
budget = LAST_MSG_BUDGET if index == last_index else MAX_CHARS
|
||||
content = compact(message["content"], budget)
|
||||
if len(content) > 200:
|
||||
digest = hashlib.md5(content.encode("utf-8")).hexdigest()
|
||||
if digest in seen:
|
||||
content = "[same as previous tool result]"
|
||||
else:
|
||||
seen.add(digest)
|
||||
message["content"] = content
|
||||
return merged
|
||||
|
||||
|
||||
def read_acp_events(path: Path) -> list[dict[str, Any]]:
|
||||
events: list[dict[str, Any]] = []
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line_number, line in enumerate(handle, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSON on line {line_number}: {exc.msg}") from exc
|
||||
if not isinstance(event, dict):
|
||||
raise ValueError(f"line {line_number} is not a JSON object")
|
||||
events.append(event)
|
||||
return events
|
||||
|
||||
|
||||
def read_acp_state(path: Path) -> list[dict[str, str]]:
|
||||
return acp_events_to_state(read_acp_events(path))
|
||||
@@ -0,0 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .paths import ENV_FILE
|
||||
|
||||
|
||||
def load_project_env() -> None:
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except ImportError:
|
||||
return
|
||||
if ENV_FILE.is_file():
|
||||
load_dotenv(ENV_FILE, override=False)
|
||||
|
||||
|
||||
def atomic_write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="") as handle:
|
||||
handle.write(text)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp, path)
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
os.unlink(tmp)
|
||||
|
||||
|
||||
def atomic_write_json(path: Path, value: Any) -> None:
|
||||
atomic_write_text(path, json.dumps(value, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
|
||||
def atomic_write_jsonl(path: Path, rows: Iterable[dict[str, Any]]) -> None:
|
||||
atomic_write_text(path, "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows))
|
||||
|
||||
|
||||
def load_json(path: Path, default: Any = None) -> Any:
|
||||
if not path.exists():
|
||||
return default
|
||||
with path.open(encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows: list[dict[str, Any]] = []
|
||||
with path.open(encoding="utf-8") as handle:
|
||||
for line_no, line in enumerate(handle, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
value = json.loads(line)
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{path}:{line_no}: expected a JSON object")
|
||||
rows.append(value)
|
||||
return rows
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
return sha256_bytes(path.read_bytes())
|
||||
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
return sha256_bytes(text.encode("utf-8"))
|
||||
|
||||
|
||||
def package_manifest(root: Path) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"path": str(p.relative_to(root)), "sha256": sha256_file(p), "bytes": p.stat().st_size}
|
||||
for p in sorted(root.rglob("*"))
|
||||
if p.is_file()
|
||||
]
|
||||
|
||||
|
||||
def package_hash(root: Path) -> str:
|
||||
payload = json.dumps(package_manifest(root), sort_keys=True, separators=(",", ":"))
|
||||
return sha256_text(payload)
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..models import RolloutTrace
|
||||
from ..scoring.state import acp_events_to_state, read_acp_events
|
||||
|
||||
|
||||
VARIANT_TO_COMPILE_TYPE = {
|
||||
"model_skill": "model_compile",
|
||||
"ori_skill": "ori",
|
||||
}
|
||||
|
||||
|
||||
def event_to_state(trajectory_path: Path) -> tuple[list[dict[str, str]], bool]:
|
||||
events = read_acp_events(trajectory_path)
|
||||
skill_invoked = any(
|
||||
event.get("type") == "tool_call"
|
||||
and any(
|
||||
str(event.get(field, "")).strip().lower() == "skill"
|
||||
for field in ("title", "kind")
|
||||
)
|
||||
for event in events
|
||||
)
|
||||
return acp_events_to_state(events), skill_invoked
|
||||
|
||||
|
||||
def trajectory_for_test(test_dir: Path) -> Path | None:
|
||||
candidates = sorted(test_dir.rglob("acp_trajectory.jsonl"))
|
||||
if not candidates:
|
||||
return None
|
||||
canonical = [path for path in candidates if "trajectory" in path.parts]
|
||||
return canonical[0] if canonical else candidates[0]
|
||||
|
||||
|
||||
def _result_for_trajectory(trajectory_path: Path) -> dict[str, Any]:
|
||||
result_path = trajectory_path.parent.parent / "result.json"
|
||||
if not result_path.is_file():
|
||||
raise ValueError(f"missing structured BenchFlow result: {result_path}")
|
||||
value = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"BenchFlow result must be a JSON object: {result_path}")
|
||||
return value
|
||||
|
||||
|
||||
def load_benchflow_trace(
|
||||
test_dir: Path,
|
||||
task_name: str,
|
||||
compile_type: str,
|
||||
) -> RolloutTrace:
|
||||
trajectory = trajectory_for_test(test_dir)
|
||||
if trajectory is None:
|
||||
raise ValueError(f"missing acp_trajectory.jsonl under {test_dir}")
|
||||
result = _result_for_trajectory(trajectory)
|
||||
agent_timeout = result.get("agent_timeout_info")
|
||||
idle_timeout = result.get("idle_timeout_info")
|
||||
timeout_info = (
|
||||
agent_timeout
|
||||
if isinstance(agent_timeout, dict)
|
||||
else idle_timeout if isinstance(idle_timeout, dict) else None
|
||||
)
|
||||
timed_out = timeout_info is not None
|
||||
metadata: dict[str, Any] = {
|
||||
"source": str(test_dir.resolve()),
|
||||
"termination": "timeout" if timed_out else "completed",
|
||||
}
|
||||
timing = result.get("timing") if isinstance(result.get("timing"), dict) else {}
|
||||
execution_seconds = timing.get("agent_execution")
|
||||
if execution_seconds is None and isinstance(timeout_info, dict):
|
||||
execution_seconds = timeout_info.get(
|
||||
"wall_clock_elapsed_sec", timeout_info.get("timeout_sec")
|
||||
)
|
||||
if isinstance(execution_seconds, (int, float)):
|
||||
metadata["agent_execution_seconds"] = float(execution_seconds)
|
||||
if isinstance(result.get("n_tool_calls"), int):
|
||||
metadata["tool_calls"] = result["n_tool_calls"]
|
||||
if timed_out:
|
||||
metadata["timeout_reason"] = timeout_info.get("reason")
|
||||
metadata["timeout_seconds"] = timeout_info.get(
|
||||
"timeout_sec", timeout_info.get("idle_timeout_sec")
|
||||
)
|
||||
metadata["partial_trajectory"] = bool(result.get("partial_trajectory", False))
|
||||
metadata["error_category"] = result.get("error_category")
|
||||
state, skill_invoked = event_to_state(trajectory)
|
||||
return RolloutTrace(
|
||||
trace_id=f"{task_name}/{compile_type}/{test_dir.name}",
|
||||
task_name=task_name,
|
||||
compile_type=compile_type,
|
||||
test_name=test_dir.name,
|
||||
state=state,
|
||||
skill_invoked=skill_invoked,
|
||||
timed_out=timed_out,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
def _variant_dirs(input_path: Path) -> list[tuple[Path, str, str]]:
|
||||
if input_path.name in VARIANT_TO_COMPILE_TYPE:
|
||||
return [(
|
||||
input_path,
|
||||
input_path.parent.name,
|
||||
VARIANT_TO_COMPILE_TYPE[input_path.name],
|
||||
)]
|
||||
direct_variants = [
|
||||
(input_path / variant_name, input_path.name, compile_type)
|
||||
for variant_name, compile_type in VARIANT_TO_COMPILE_TYPE.items()
|
||||
if (input_path / variant_name).is_dir()
|
||||
]
|
||||
if direct_variants:
|
||||
return direct_variants
|
||||
variants = []
|
||||
for task_dir in sorted(path for path in input_path.iterdir() if path.is_dir()):
|
||||
for variant_name, compile_type in VARIANT_TO_COMPILE_TYPE.items():
|
||||
variant_dir = task_dir / variant_name
|
||||
if variant_dir.is_dir():
|
||||
variants.append((variant_dir, task_dir.name, compile_type))
|
||||
return variants
|
||||
|
||||
|
||||
def load_benchflow_traces(input_path: Path) -> list[RolloutTrace]:
|
||||
input_path = input_path.resolve()
|
||||
if not input_path.is_dir():
|
||||
raise ValueError(f"BenchFlow input does not exist: {input_path}")
|
||||
variants = _variant_dirs(input_path)
|
||||
if not variants:
|
||||
raise ValueError(f"no model_skill or ori_skill directories under {input_path}")
|
||||
traces = []
|
||||
for variant_dir, task_name, compile_type in variants:
|
||||
for test_dir in sorted(path for path in variant_dir.glob("test-*") if path.is_dir()):
|
||||
traces.append(load_benchflow_trace(test_dir, task_name, compile_type))
|
||||
return traces
|
||||
Reference in New Issue
Block a user